# HG changeset patch # User Marcin Kuzminski # Date 1285546623 -7200 # Node ID d5efb83590ef3f05f2db5af1cde89e75aec507fe # Parent 5aba7adff1960ab81bec2b62dc673b00ab2255ca fixed lock decorator bug which didn't release the lock after func execution and rewrote the pidlock a little with Ask Solem suggestions! added tredning languages stats diff -r 5aba7adff196 -r d5efb83590ef celeryconfig.py --- a/celeryconfig.py Sat Sep 25 23:36:10 2010 +0200 +++ b/celeryconfig.py Mon Sep 27 02:17:03 2010 +0200 @@ -36,7 +36,7 @@ CELERYD_CONCURRENCY = 2 # CELERYD_LOG_FILE = "celeryd.log" CELERYD_LOG_LEVEL = "DEBUG" -CELERYD_MAX_TASKS_PER_CHILD = 1 +CELERYD_MAX_TASKS_PER_CHILD = 3 #Tasks will never be sent to the queue, but executed locally instead. CELERY_ALWAYS_EAGER = False diff -r 5aba7adff196 -r d5efb83590ef pylons_app/controllers/summary.py --- a/pylons_app/controllers/summary.py Sat Sep 25 23:36:10 2010 +0200 +++ b/pylons_app/controllers/summary.py Mon Sep 27 02:17:03 2010 +0200 @@ -35,7 +35,7 @@ from time import mktime import calendar import logging - +import json log = logging.getLogger(__name__) class SummaryController(BaseController): @@ -79,18 +79,25 @@ c.ts_min = ts_min_m c.ts_max = ts_max_y - stats = self.sa.query(Statistics)\ .filter(Statistics.repository == c.repo_info.dbrepo)\ .scalar() - - if stats: + + + if stats and stats.languages: + lang_stats = json.loads(stats.languages) c.commit_data = stats.commit_activity c.overview_data = stats.commit_activity_combined + c.trending_languages = json.dumps(OrderedDict( + sorted(lang_stats.items(), reverse=True, + key=lambda k: k[1])[:2] + ) + ) + print c.trending_languages else: - import json c.commit_data = json.dumps({}) c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ]) + c.trending_languages = json.dumps({}) return render('summary/summary.html') diff -r 5aba7adff196 -r d5efb83590ef pylons_app/lib/celerylib/__init__.py --- a/pylons_app/lib/celerylib/__init__.py Sat Sep 25 23:36:10 2010 +0200 +++ b/pylons_app/lib/celerylib/__init__.py Mon Sep 27 02:17:03 2010 +0200 @@ -42,7 +42,7 @@ log.info('running task with lockkey %s', lockkey) try: l = DaemonLock(lockkey) - return func(*fargs, **fkwargs) + func(*fargs, **fkwargs) l.release() except LockHeld: log.info('LockHeld') diff -r 5aba7adff196 -r d5efb83590ef pylons_app/lib/celerylib/tasks.py --- a/pylons_app/lib/celerylib/tasks.py Sat Sep 25 23:36:10 2010 +0200 +++ b/pylons_app/lib/celerylib/tasks.py Mon Sep 27 02:17:03 2010 +0200 @@ -1,16 +1,16 @@ from celery.decorators import task from celery.task.sets import subtask from celeryconfig import PYLONS_CONFIG as config +from operator import itemgetter from pylons.i18n.translation import _ from pylons_app.lib.celerylib import run_task, locked_task from pylons_app.lib.helpers import person from pylons_app.lib.smtp_mailer import SmtpMailer from pylons_app.lib.utils import OrderedDict -from operator import itemgetter +from time import mktime from vcs.backends.hg import MercurialRepository -from time import mktime +import json import traceback -import json __all__ = ['whoosh_index', 'get_commits_stats', 'reset_user_password', 'send_email'] @@ -75,10 +75,10 @@ @task @locked_task def get_commits_stats(repo_name, ts_min_y, ts_max_y): - author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty - from pylons_app.model.db import Statistics, Repository log = get_commits_stats.get_logger() + author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty + commits_by_day_author_aggregate = {} commits_by_day_aggregate = {} repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') @@ -99,7 +99,7 @@ if cur_stats: last_rev = cur_stats.stat_on_revision - if last_rev == repo.revisions[-1]: + if last_rev == repo.revisions[-1] and len(repo.revisions) > 1: #pass silently without any work return True @@ -109,6 +109,7 @@ cur_stats.commit_activity_combined)) commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity) + log.debug('starting parsing %s', parse_limit) for cnt, rev in enumerate(repo.revisions[last_rev:]): last_cs = cs = repo.get_changeset(rev) k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1], @@ -187,9 +188,16 @@ stats = cur_stats if cur_stats else Statistics() stats.commit_activity = json.dumps(commits_by_day_author_aggregate) stats.commit_activity_combined = json.dumps(overview_data) + + log.debug('last revison %s', last_rev) + leftovers = len(repo.revisions[last_rev:]) + log.debug('revisions to parse %s', leftovers) + + if last_rev == 0 or leftovers < parse_limit: + stats.languages = json.dumps(__get_codes_stats(repo_name)) + stats.repository = dbrepo stats.stat_on_revision = last_cs.revision - stats.languages = json.dumps({'_TOTAL_':0, '':0}) try: sa.add(stats) @@ -198,8 +206,8 @@ log.error(traceback.format_exc()) sa.rollback() return False - - run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) + if len(repo.revisions) > 1: + run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) return True @@ -259,3 +267,31 @@ log.error(traceback.format_exc()) return False return True + +def __get_codes_stats(repo_name): + LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', + 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', 'erl', + 'h', 'java', 'js', 'jsp', 'jspx', 'lisp', + 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', + 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', + 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', + 'yaws'] + repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') + repo = MercurialRepository(repos_path + repo_name) + + code_stats = {} + for topnode, dirs, files in repo.walk('/', 'tip'): + for f in files: + k = f.mimetype + if f.extension in LANGUAGES_EXTENSIONS: + if code_stats.has_key(k): + code_stats[k] += 1 + else: + code_stats[k] = 1 + + return code_stats or {} + + + + + diff -r 5aba7adff196 -r d5efb83590ef pylons_app/lib/pidlock.py --- a/pylons_app/lib/pidlock.py Sat Sep 25 23:36:10 2010 +0200 +++ b/pylons_app/lib/pidlock.py Mon Sep 27 02:17:03 2010 +0200 @@ -1,6 +1,8 @@ import os, time import sys from warnings import warn +from multiprocessing.util import Finalize +import errno class LockHeld(Exception):pass @@ -27,55 +29,67 @@ self.held = False #run the lock automatically ! self.lock() - - def __del__(self): - if self.held: + self._finalize = Finalize(self, DaemonLock._on_finalize, + args=(self, debug), exitpriority=10) -# warn("use lock.release instead of del lock", -# category = DeprecationWarning, -# stacklevel = 2) - - # ensure the lock will be removed - self.release() + @staticmethod + def _on_finalize(lock, debug): + if lock.held: + if debug: + print 'leck held finilazing and running lock.release()' + lock.release() def lock(self): """locking function, if lock is present it will raise LockHeld exception """ lockname = '%s' % (os.getpid()) - + if self.debug: + print 'running lock' self.trylock() self.makelock(lockname, self.pidfile) return True def trylock(self): running_pid = False + if self.debug: + print 'checking for already running process' try: pidfile = open(self.pidfile, "r") pidfile.seek(0) - running_pid = pidfile.readline() + running_pid = int(pidfile.readline()) + + pidfile.close() + if self.debug: print 'lock file present running_pid: %s, checking for execution'\ % running_pid # Now we check the PID from lock file matches to the current # process PID if running_pid: - if os.path.exists("/proc/%s" % running_pid): - print "You already have an instance of the program running" - print "It is running as process %s" % running_pid - raise LockHeld + try: + os.kill(running_pid, 0) + except OSError, exc: + if exc.errno in (errno.ESRCH, errno.EPERM): + print "Lock File is there but the program is not running" + print "Removing lock file for the: %s" % running_pid + self.release() + raise else: - print "Lock File is there but the program is not running" - print "Removing lock file for the: %s" % running_pid - self.release() + print "You already have an instance of the program running" + print "It is running as process %s" % running_pid + raise LockHeld() + except IOError, e: if e.errno != 2: raise - def release(self): """releases the pid by removing the pidfile """ + if self.debug: + print 'trying to release the pidlock' + if self.callbackfn: #execute callback function on release if self.debug: diff -r 5aba7adff196 -r d5efb83590ef pylons_app/public/css/style.css --- a/pylons_app/public/css/style.css Sat Sep 25 23:36:10 2010 +0200 +++ b/pylons_app/public/css/style.css Mon Sep 27 02:17:03 2010 +0200 @@ -2963,6 +2963,28 @@ /* ----------------------------------------------------------- SUMMARY ----------------------------------------------------------- */ +.trending_language_tbl, .trending_language_tbl td { + margin: 0px !important; + padding: 0px !important; + border: 0 !important; + +} +.trending_language{ + -moz-border-radius-bottomright:4px; + -moz-border-radius-topright:4px; + border-bottom-right-radius: 4px 4px; + border-top-right-radius: 4px 4px; + background-color:#336699; + color:#FFFFFF; + display:block; + min-width:20px; + max-width:400px; + padding:3px; + text-decoration:none; + height: 10px; + margin-bottom: 4px; + margin-left: 5px; +} #clone_url{ border: none; diff -r 5aba7adff196 -r d5efb83590ef pylons_app/templates/summary/summary.html --- a/pylons_app/templates/summary/summary.html Sat Sep 25 23:36:10 2010 +0200 +++ b/pylons_app/templates/summary/summary.html Mon Sep 27 02:17:03 2010 +0200 @@ -90,7 +90,62 @@ - + +
+
+ +
+
+
+ +
+ + +
+
+