# HG changeset patch # User Marcin Kuzminski # Date 1275516288 -7200 # Node ID a83a1799480c91cf3a73320ff0042ffc95dc459d # Parent 782f0692b29ccb6efaa5e779c3c22a64895b4724 Reimplemented way of caching repos list, hg model now get's repos objects right from cached dict, this way we skipp creating instances of MercurialRepository and gain performance. Some import cleanup diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/branches.py --- a/pylons_app/controllers/branches.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/branches.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,15 +1,11 @@ +from pylons import tmpl_context as c, app_globals as g +from pylons_app.lib.auth import LoginRequired +from pylons_app.lib.base import BaseController, render +from pylons_app.model.hg_model import HgModel import logging -from pylons import tmpl_context as c, app_globals as g, session, request, config, url -from pylons.controllers.util import abort, redirect - -from pylons_app.lib.base import BaseController, render -from pylons_app.lib.utils import get_repo_slug -from pylons_app.model.hg_model import HgModel -from pylons_app.lib.auth import LoginRequired log = logging.getLogger(__name__) - class BranchesController(BaseController): @LoginRequired() diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/changelog.py --- a/pylons_app/controllers/changelog.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/changelog.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,11 +1,10 @@ from mercurial.graphmod import revisions as graph_rev, colored, CHANGESET from mercurial.node import short -from pylons import request, response, session, tmpl_context as c, url, config, \ - app_globals as g -from pylons.controllers.util import abort, redirect +from pylons import request, session, tmpl_context as c from pylons_app.lib.auth import LoginRequired -from pylons_app.lib.base import BaseController, render, _full_changelog_cached +from pylons_app.lib.base import BaseController, render from pylons_app.lib.filters import age as _age, person +from pylons_app.model.hg_model import _full_changelog_cached from simplejson import dumps from webhelpers.paginate import Page import logging diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/changeset.py --- a/pylons_app/controllers/changeset.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/changeset.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,6 +1,4 @@ -from pylons import request, response, session, tmpl_context as c, url, config, \ - app_globals as g -from pylons.controllers.util import abort, redirect +from pylons import tmpl_context as c from pylons_app.lib.auth import LoginRequired from pylons_app.lib.base import BaseController, render from pylons_app.model.hg_model import HgModel diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/feed.py --- a/pylons_app/controllers/feed.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/feed.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,13 +1,10 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +from pylons import tmpl_context as c, url, response +from pylons_app.lib.base import BaseController, render +from pylons_app.model.hg_model import _full_changelog_cached +from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed import logging -from operator import itemgetter -from pylons import tmpl_context as c, request, config, url, response -from pylons_app.lib.base import BaseController, render, _full_changelog_cached -from pylons_app.lib.utils import get_repo_slug -from pylons_app.model.hg_model import HgModel -from pylons_app.lib.auth import LoginRequired -from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed log = logging.getLogger(__name__) class FeedController(BaseController): @@ -35,8 +32,9 @@ if cnt > self.feed_nr: break feed.add_item(title=cs.message, - link=url('changeset_home', repo_name=repo_name, revision=cs.raw_id, qualified=True), - description=str(cs.date)) + link=url('changeset_home', repo_name=repo_name, + revision=cs.raw_id, qualified=True), + description=str(cs.date)) response.content_type = feed.mime_type return feed.writeString('utf-8') diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/files.py --- a/pylons_app/controllers/files.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/files.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,15 +1,12 @@ -import tempfile -from pylons import request, response, session, tmpl_context as c, url, config, \ - app_globals as g -from pylons.controllers.util import abort, redirect +from mercurial import archival +from pylons import request, response, session, tmpl_context as c, url from pylons_app.lib.auth import LoginRequired from pylons_app.lib.base import BaseController, render -from pylons_app.lib.utils import get_repo_slug from pylons_app.model.hg_model import HgModel from vcs.exceptions import RepositoryError, ChangesetError from vcs.utils import diffs as differ import logging -from mercurial import archival +import tempfile log = logging.getLogger(__name__) diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/hg.py --- a/pylons_app/controllers/hg.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/hg.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,10 +1,11 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -import logging from operator import itemgetter from pylons import tmpl_context as c, request, config +from pylons_app.lib.auth import LoginRequired from pylons_app.lib.base import BaseController, render -from pylons_app.lib.auth import LoginRequired +from pylons_app.model.hg_model import HgModel +import logging log = logging.getLogger(__name__) class HgController(BaseController): @@ -18,12 +19,12 @@ cs = c.current_sort c.cs_slug = cs.replace('-', '') sortables = ['name', 'description', 'last_change', 'tip', 'contact'] - + cached_repo_list = HgModel().get_repos() if cs and c.cs_slug in sortables: sort_key = c.cs_slug + '_sort' if cs.startswith('-'): - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), reverse=True) + c.repos_list = sorted(cached_repo_list, key=itemgetter(sort_key), reverse=True) else: - c.repos_list = sorted(c.cached_repo_list, key=itemgetter(sort_key), reverse=False) + c.repos_list = sorted(cached_repo_list, key=itemgetter(sort_key), reverse=False) return render('/index.html') diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/shortlog.py --- a/pylons_app/controllers/shortlog.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/shortlog.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,13 +1,9 @@ -import logging - -from pylons import tmpl_context as c, app_globals as g, session, request, config, url -from pylons.controllers.util import abort, redirect - +from pylons import tmpl_context as c, request +from pylons_app.lib.auth import LoginRequired from pylons_app.lib.base import BaseController, render -from pylons_app.lib.utils import get_repo_slug from pylons_app.model.hg_model import HgModel from webhelpers.paginate import Page -from pylons_app.lib.auth import LoginRequired +import logging log = logging.getLogger(__name__) diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/summary.py --- a/pylons_app/controllers/summary.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/summary.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,7 +1,7 @@ from pylons import tmpl_context as c, request from pylons_app.lib.auth import LoginRequired -from pylons_app.lib.base import BaseController, render, _full_changelog_cached -from pylons_app.model.hg_model import HgModel +from pylons_app.lib.base import BaseController, render +from pylons_app.model.hg_model import HgModel, _full_changelog_cached import logging log = logging.getLogger(__name__) @@ -16,7 +16,6 @@ hg_model = HgModel() c.repo_info = hg_model.get_repo(c.repo_name) c.repo_changesets = _full_changelog_cached(c.repo_name)[:10] - e = request.environ uri = u'%(protocol)s://%(user)s@%(host)s/%(repo_name)s' % { 'protocol': e.get('wsgi.url_scheme'), diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/tags.py --- a/pylons_app/controllers/tags.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/tags.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,15 +1,11 @@ +from pylons import tmpl_context as c +from pylons_app.lib.auth import LoginRequired +from pylons_app.lib.base import BaseController, render +from pylons_app.model.hg_model import HgModel import logging -from pylons import tmpl_context as c, app_globals as g, session, request, config, url -from pylons.controllers.util import abort, redirect - -from pylons_app.lib.base import BaseController, render -from pylons_app.lib.utils import get_repo_slug -from pylons_app.model.hg_model import HgModel -from pylons_app.lib.auth import LoginRequired log = logging.getLogger(__name__) - class TagsController(BaseController): @LoginRequired() diff -r 782f0692b29c -r a83a1799480c pylons_app/controllers/users.py --- a/pylons_app/controllers/users.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/controllers/users.py Thu Jun 03 00:04:48 2010 +0200 @@ -1,9 +1,8 @@ from formencode import htmlfill -from pylons import request, response, session, tmpl_context as c, url, \ - app_globals as g +from pylons import request, session, tmpl_context as c, url +from pylons.controllers.util import abort, redirect from pylons.i18n.translation import _ -from pylons_app.lib import helpers as h -from pylons.controllers.util import abort, redirect +from pylons_app.lib import helpers as h from pylons_app.lib.auth import LoginRequired, CheckPermissionAll from pylons_app.lib.base import BaseController, render from pylons_app.model.db import User, UserLog @@ -12,8 +11,6 @@ import formencode import logging - - log = logging.getLogger(__name__) class UsersController(BaseController): diff -r 782f0692b29c -r a83a1799480c pylons_app/lib/base.py --- a/pylons_app/lib/base.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/lib/base.py Thu Jun 03 00:04:48 2010 +0200 @@ -2,24 +2,15 @@ Provides the BaseController class for subclassing. """ -from beaker.cache import cache_region from pylons import config, tmpl_context as c, request, session from pylons.controllers import WSGIController from pylons.templating import render_mako as render from pylons_app.lib.auth import LoginRequired, AuthUser from pylons_app.lib.utils import get_repo_slug from pylons_app.model import meta -from pylons_app.model.hg_model import HgModel +from pylons_app.model.hg_model import _get_repos_cached from pylons_app import __version__ -@cache_region('long_term', 'cached_repo_list') -def _get_repos_cached(): - return [rep for rep in HgModel().get_repos()] - -@cache_region('long_term', 'full_changelog') -def _full_changelog_cached(repo_name): - return list(reversed(list(HgModel().get_repo(repo_name)))) - class BaseController(WSGIController): def __before__(self): diff -r 782f0692b29c -r a83a1799480c pylons_app/lib/utils.py --- a/pylons_app/lib/utils.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/lib/utils.py Thu Jun 03 00:04:48 2010 +0200 @@ -107,11 +107,11 @@ args = tuple(tmp) if name == 'cached_repo_list': - from pylons_app.lib.base import _get_repos_cached + from pylons_app.model.hg_model import _get_repos_cached region_invalidate(_get_repos_cached, None, *args) if name == 'full_changelog': - from pylons_app.lib.base import _full_changelog_cached + from pylons_app.model.hg_model import _full_changelog_cached region_invalidate(_full_changelog_cached, None, *args) from vcs.backends.base import BaseChangeset @@ -128,3 +128,17 @@ """ return '0' * 12 + +def repo2db_mapper(): + """ + put ! + """ + pass + #scann all dirs for .hgdbid + #if some dir doesn't have one generate one. + # + + + + + diff -r 782f0692b29c -r a83a1799480c pylons_app/model/hg_model.py --- a/pylons_app/model/hg_model.py Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/model/hg_model.py Thu Jun 03 00:04:48 2010 +0200 @@ -3,65 +3,118 @@ # # Copyright (c) 2010 marcink. All rights reserved. # -from vcs.exceptions import RepositoryError ''' Created on Apr 9, 2010 @author: marcink ''' + +from beaker.cache import cache_region +from mercurial import ui +from mercurial.hgweb.hgwebdir_mod import findrepos +from pylons import app_globals as g +from vcs.exceptions import RepositoryError, VCSError +import logging import os -from pylons import tmpl_context as c, app_globals as g, session, request, config -from pylons.controllers.util import abort import sys +log = logging.getLogger(__name__) + try: - from vcs.backends.hg import get_repositories, MercurialRepository + from vcs.backends.hg import MercurialRepository except ImportError: sys.stderr.write('You have to import vcs module') raise Exception('Unable to import vcs') + +@cache_region('long_term', 'cached_repo_list') +def _get_repos_cached(): + """ + return cached dict with repos + """ + return HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui) + +@cache_region('long_term', 'full_changelog') +def _full_changelog_cached(repo_name): + log.info('getting full changelog for %s', repo_name) + return list(reversed(list(HgModel().get_repo(repo_name)))) + class HgModel(object): """ Mercurial Model """ - def __init__(self): """ Constructor """ pass - + + @staticmethod + def repo_scan(repos_prefix, repos_path, baseui): + """ + Listing of repositories in given path. This path should not be a + repository itself. Return a dictionary of repository objects + :param repos_path: path to directory it could take syntax with + * or ** for deep recursive displaying repositories + """ + def check_repo_dir(path): + """ + Checks the repository + :param path: + """ + repos_path = path.split('/') + if repos_path[-1] in ['*', '**']: + repos_path = repos_path[:-1] + if repos_path[0] != '/': + repos_path[0] = '/' + if not os.path.isdir(os.path.join(*repos_path)): + raise RepositoryError('Not a valid repository in %s' % path[0][1]) + if not repos_path.endswith('*'): + raise VCSError('You need to specify * or ** at the end of path ' + 'for recursive scanning') + + check_repo_dir(repos_path) + log.info('scanning for repositories in %s', repos_path) + repos = findrepos([(repos_prefix, repos_path)]) + if not isinstance(baseui, ui.ui): + baseui = ui.ui() + + repos_list = {} + for name, path in repos: + try: + repos_list[name] = MercurialRepository(path, baseui=baseui) + except OSError: + continue + return repos_list + def get_repos(self): - for mercurial_repo in get_repositories(g.paths[0][0], g.paths[0][1], g.baseui): - - if mercurial_repo._get_hidden(): + for name, repo in _get_repos_cached().items(): + if repo._get_hidden(): #skip hidden web repository continue - last_change = mercurial_repo.last_change + last_change = repo.last_change try: - tip = mercurial_repo.get_changeset('tip') + tip = repo.get_changeset('tip') except RepositoryError: from pylons_app.lib.utils import EmptyChangeset tip = EmptyChangeset() tmp_d = {} - tmp_d['name'] = mercurial_repo.name + tmp_d['name'] = repo.name tmp_d['name_sort'] = tmp_d['name'].lower() - tmp_d['description'] = mercurial_repo.description + tmp_d['description'] = repo.description tmp_d['description_sort'] = tmp_d['description'] tmp_d['last_change'] = last_change tmp_d['last_change_sort'] = last_change[1] - last_change[0] tmp_d['tip'] = tip.raw_id tmp_d['tip_sort'] = tip.revision tmp_d['rev'] = tip.revision - tmp_d['contact'] = mercurial_repo.contact + tmp_d['contact'] = repo.contact tmp_d['contact_sort'] = tmp_d['contact'] - tmp_d['repo_archives'] = list(mercurial_repo._get_archives()) + tmp_d['repo_archives'] = list(repo._get_archives()) yield tmp_d def get_repo(self, repo_name): - path = g.paths[0][1].replace('*', '') - repo = MercurialRepository(os.path.join(path, repo_name), baseui=g.baseui) - return repo + return _get_repos_cached()[repo_name] diff -r 782f0692b29c -r a83a1799480c pylons_app/templates/base/base.html --- a/pylons_app/templates/base/base.html Tue Jun 01 22:19:03 2010 +0200 +++ b/pylons_app/templates/base/base.html Thu Jun 03 00:04:48 2010 +0200 @@ -92,8 +92,8 @@ ↓