Mercurial > kallithea
changeset 665:070f32743632 beta
Moved out reposcan into hg Model.
Rewrote repo scann and caching of repositories, all is in hgModel.
Changed invalidate cache calls.
mergeds main repo list and repo switcher list into one new based on hgModel.
| author | Marcin Kuzminski <marcin@python-works.com> |
|---|---|
| date | Sun, 07 Nov 2010 15:02:56 +0100 |
| parents | 50d9b1afaca7 |
| children | 6ed37675e78b |
| files | rhodecode/config/environment.py rhodecode/controllers/admin/repos.py rhodecode/controllers/admin/settings.py rhodecode/controllers/settings.py rhodecode/lib/base.py rhodecode/lib/middleware/simplegit.py rhodecode/lib/middleware/simplehg.py rhodecode/lib/utils.py rhodecode/model/hg.py rhodecode/templates/admin/users/user_edit_my_account.html rhodecode/templates/base/base.html |
| diffstat | 11 files changed, 151 insertions(+), 180 deletions(-) [+] |
line wrap: on
line diff
--- a/rhodecode/config/environment.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/config/environment.py Sun Nov 07 15:02:56 2010 +0100 @@ -6,7 +6,7 @@ from rhodecode.lib.auth import set_available_permissions, set_base_path from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config from rhodecode.model import init_model -from rhodecode.model.hg import _get_repos_cached_initial +from rhodecode.model.hg import HgModel from sqlalchemy import engine_from_config import logging import os @@ -69,7 +69,8 @@ #init baseui config['pylons.app_globals'].baseui = make_ui('db') - repo2db_mapper(_get_repos_cached_initial(config['pylons.app_globals'], initial)) + g = config['pylons.app_globals'] + repo2db_mapper(HgModel().repo_scan(g.paths[0][1], g.baseui, initial)) set_available_permissions(config) set_base_path(config) set_rhodecode_config(config)
--- a/rhodecode/controllers/admin/repos.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/controllers/admin/repos.py Sun Nov 07 15:02:56 2010 +0100 @@ -74,7 +74,6 @@ try: form_result = _form.to_python(dict(request.POST)) repo_model.create(form_result, c.rhodecode_user) - invalidate_cache('cached_repo_list') h.flash(_('created repository %s') % form_result['repo_name'], category='success') @@ -133,7 +132,7 @@ try: form_result = _form.to_python(dict(request.POST)) repo_model.update(repo_name, form_result) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('Repository %s updated succesfully' % repo_name), category='success') changed_name = form_result['repo_name'] @@ -182,7 +181,7 @@ action_logger(self.rhodecode_user, 'admin_deleted_repo', repo_name, '', self.sa) repo_model.delete(repo) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('deleted repository %s') % repo_name, category='success') except Exception, e:
--- a/rhodecode/controllers/admin/settings.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/controllers/admin/settings.py Sun Nov 07 15:02:56 2010 +0100 @@ -33,12 +33,13 @@ from rhodecode.lib.base import BaseController, render from rhodecode.lib.utils import repo2db_mapper, invalidate_cache, \ set_rhodecode_config, get_hg_settings, get_hg_ui_settings -from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi +from rhodecode.model.db import RhodeCodeSettings, RhodeCodeUi, Repository from rhodecode.model.forms import UserForm, ApplicationSettingsForm, \ ApplicationUiSettingsForm from rhodecode.model.hg import HgModel from rhodecode.model.user import UserModel from rhodecode.lib.celerylib import tasks, run_task +from sqlalchemy import func import formencode import logging import traceback @@ -98,9 +99,12 @@ rm_obsolete = request.POST.get('destroy', False) log.debug('Rescanning directories with destroy=%s', rm_obsolete) - initial = HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui) + initial = HgModel().repo_scan(g.paths[0][1], g.baseui) + for repo_name in initial.keys(): + invalidate_cache('get_repo_cached_%s' % repo_name) + repo2db_mapper(initial, rm_obsolete) - invalidate_cache('cached_repo_list') + h.flash(_('Repositories successfully rescanned'), category='success') if setting_id == 'whoosh': @@ -238,12 +242,14 @@ """ GET /_admin/my_account Displays info about my account """ + # url('admin_settings_my_account') c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False) - c.user_repos = [] - for repo in c.cached_repo_list.values(): - if repo.dbrepo.user.username == c.user.username: - c.user_repos.append(repo) + all_repos = self.sa.query(Repository)\ + .filter(Repository.user_id == c.user.user_id)\ + .order_by(func.lower(Repository.repo_name))\ + .all() + c.user_repos = HgModel().get_repos(all_repos) if c.user.username == 'default': h.flash(_("You can't edit this user since it's"
--- a/rhodecode/controllers/settings.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/controllers/settings.py Sun Nov 07 15:02:56 2010 +0100 @@ -78,7 +78,7 @@ try: form_result = _form.to_python(dict(request.POST)) repo_model.update(repo_name, form_result) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('Repository %s updated successfully' % repo_name), category='success') changed_name = form_result['repo_name'] @@ -96,7 +96,7 @@ encoding="UTF-8") except Exception: log.error(traceback.format_exc()) - h.flash(_('error occured during update of repository %s') \ + h.flash(_('error occurred during update of repository %s') \ % repo_name, category='error') return redirect(url('repo_settings_home', repo_name=changed_name)) @@ -126,7 +126,7 @@ action_logger(self.rhodecode_user, 'user_deleted_repo', repo_name, '', self.sa) repo_model.delete(repo) - invalidate_cache('cached_repo_list') + invalidate_cache('get_repo_cached_%s' % repo_name) h.flash(_('deleted repository %s') % repo_name, category='success') except Exception: h.flash(_('An error occurred during deletion of %s') % repo_name,
--- a/rhodecode/lib/base.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/lib/base.py Sun Nov 07 15:02:56 2010 +0100 @@ -9,20 +9,20 @@ from rhodecode.lib import auth from rhodecode.lib.utils import get_repo_slug from rhodecode.model import meta -from rhodecode.model.hg import _get_repos_cached, \ - _get_repos_switcher_cached +from rhodecode.model.hg import HgModel from vcs import BACKENDS + class BaseController(WSGIController): def __before__(self): c.rhodecode_version = __version__ c.rhodecode_name = config['rhodecode_title'] c.repo_name = get_repo_slug(request) - c.cached_repo_list = _get_repos_cached() - c.repo_switcher_list = _get_repos_switcher_cached(c.cached_repo_list) + c.cached_repo_list = HgModel().get_repos() c.backends = BACKENDS.keys() + if c.repo_name: - cached_repo = c.cached_repo_list.get(c.repo_name) + cached_repo = HgModel().get(c.repo_name) if cached_repo: c.repository_tags = cached_repo.tags
--- a/rhodecode/lib/middleware/simplegit.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/lib/middleware/simplegit.py Sun Nov 07 15:02:56 2010 +0100 @@ -83,15 +83,15 @@ self.repository = None self.username = None self.action = None - + def __call__(self, environ, start_response): if not is_git(environ): return self.application(environ, start_response) - + proxy_key = 'HTTP_X_REAL_IP' def_key = 'REMOTE_ADDR' self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0')) - + #=================================================================== # AUTHENTICATE THIS GIT REQUEST #=================================================================== @@ -104,7 +104,7 @@ REMOTE_USER.update(environ, result) else: return result.wsgi_application(environ, start_response) - + #======================================================================= # GET REPOSITORY #======================================================================= @@ -206,5 +206,4 @@ """we know that some change was made to repositories and we should invalidate the cache to see the changes right away but only for push requests""" - invalidate_cache('cached_repo_list') - invalidate_cache('full_changelog', repo_name) + invalidate_cache('get_repo_cached_%s' % repo_name)
--- a/rhodecode/lib/middleware/simplehg.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/lib/middleware/simplehg.py Sun Nov 07 15:02:56 2010 +0100 @@ -52,20 +52,20 @@ self.repository = None self.username = None self.action = None - + def __call__(self, environ, start_response): if not is_mercurial(environ): return self.application(environ, start_response) - + proxy_key = 'HTTP_X_REAL_IP' def_key = 'REMOTE_ADDR' self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0')) - + #=================================================================== # AUTHENTICATE THIS MERCURIAL REQUEST #=================================================================== username = REMOTE_USER(environ) - + if not username: self.authenticate.realm = self.config['rhodecode_realm'] result = self.authenticate(environ) @@ -74,7 +74,7 @@ REMOTE_USER.update(environ, result) else: return result.wsgi_application(environ, start_response) - + #======================================================================= # GET REPOSITORY #======================================================================= @@ -114,7 +114,7 @@ 'repository.admin')\ (user, repo_name): return HTTPForbidden()(environ, start_response) - + self.extras = {'ip':self.ipaddr, 'username':self.username, 'action':self.action, @@ -200,8 +200,7 @@ """we know that some change was made to repositories and we should invalidate the cache to see the changes right away but only for push requests""" - invalidate_cache('cached_repo_list') - invalidate_cache('full_changelog', repo_name) + invalidate_cache('get_repo_cached_%s' % repo_name) def __load_web_settings(self, hgserve, extras={}): @@ -209,12 +208,12 @@ hgserve.repo.ui = self.baseui hgrc = os.path.join(self.repo_path, '.hg', 'hgrc') - + #inject some additional parameters that will be available in ui #for hooks for k, v in extras.items(): hgserve.repo.ui.setconfig('rhodecode_extras', k, v) - + repoui = make_ui('file', hgrc, False) if repoui: @@ -222,7 +221,7 @@ for section in ui_sections: for k, v in repoui.configitems(section): hgserve.repo.ui.setconfig(section, k, v) - + return hgserve
--- a/rhodecode/lib/utils.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/lib/utils.py Sun Nov 07 15:02:56 2010 +0100 @@ -275,25 +275,11 @@ config[k] = v def invalidate_cache(name, *args): - """Invalidates given name cache""" - - from beaker.cache import region_invalidate - log.info('INVALIDATING CACHE FOR %s', name) - - """propagate our arguments to make sure invalidation works. First - argument has to be the name of cached func name give to cache decorator - without that the invalidation would not work""" - tmp = [name] - tmp.extend(args) - args = tuple(tmp) - - if name == 'cached_repo_list': - from rhodecode.model.hg import _get_repos_cached - region_invalidate(_get_repos_cached, None, *args) - - if name == 'full_changelog': - from rhodecode.model.hg import _full_changelog_cached - region_invalidate(_full_changelog_cached, None, *args) + """ + Puts cache invalidation task into db for + further global cache invalidation + """ + pass class EmptyChangeset(BaseChangeset): """ @@ -352,7 +338,6 @@ } rm.create(form_data, user, just_db=True) - if remove_obsolete: #remove from database those repositories that are not in the filesystem for repo in sa.query(Repository).all(): @@ -360,10 +345,6 @@ sa.delete(repo) sa.commit() - - meta.Session.remove() - - class OrderedDict(dict, DictMixin): def __init__(self, *args, **kwds):
--- a/rhodecode/model/hg.py Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/model/hg.py Sun Nov 07 15:02:56 2010 +0100 @@ -22,57 +22,25 @@ Model for RhodeCode @author: marcink """ -from beaker.cache import cache_region +from beaker.cache import cache_region, region_invalidate from mercurial import ui from rhodecode.lib import helpers as h -from rhodecode.lib.utils import invalidate_cache from rhodecode.lib.auth import HasRepoPermissionAny +from rhodecode.lib.utils import get_repos from rhodecode.model import meta -from rhodecode.model.db import Repository, User +from rhodecode.model.caching_query import FromCache +from rhodecode.model.db import Repository, User, RhodeCodeUi from sqlalchemy.orm import joinedload +from vcs import get_repo as vcs_get_repo, get_backend +from vcs.backends.hg import MercurialRepository from vcs.exceptions import RepositoryError, VCSError +from vcs.utils.lazy import LazyProperty import logging -import sys +import os import time log = logging.getLogger(__name__) -try: - from vcs.backends.hg import MercurialRepository - from vcs.backends.git import GitRepository -except ImportError: - sys.stderr.write('You have to import vcs module') - raise Exception('Unable to import vcs') - -def _get_repos_cached_initial(app_globals, initial): - """return cached dict with repos - """ - g = app_globals - return HgModel().repo_scan(g.paths[0][1], g.baseui, initial) - -@cache_region('long_term', 'cached_repo_list') -def _get_repos_cached(): - """return cached dict with repos - """ - log.info('getting all repositories list') - from pylons import app_globals as g - return HgModel().repo_scan(g.paths[0][1], g.baseui) - -@cache_region('super_short_term', 'cached_repos_switcher_list') -def _get_repos_switcher_cached(cached_repo_list): - repos_lst = [] - for repo in [x for x in cached_repo_list.values()]: - if HasRepoPermissionAny('repository.write', 'repository.read', - 'repository.admin')(repo.name, 'main page check'): - repos_lst.append((repo.name, repo.dbrepo.private,)) - - return sorted(repos_lst, key=lambda k:k[0].lower()) - -@cache_region('long_term', 'full_changelog') -def _full_changelog_cached(repo_name): - log.info('getting full changelog for %s', repo_name) - return list(reversed(list(HgModel().get_repo(repo_name)))) - class HgModel(object): """ Mercurial Model @@ -84,6 +52,16 @@ else: self.sa = sa + + @LazyProperty + def repos_path(self): + """ + Get's the repositories root path from database + """ + q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() + + return q.ui_value + def repo_scan(self, repos_path, baseui, initial=False): """ Listing of repositories in given path. This path should not be a @@ -91,93 +69,100 @@ :param repos_path: path to directory containing repositories :param baseui - :param initial: initial scann + :param initial: initial scan """ log.info('scanning for repositories in %s', repos_path) if not isinstance(baseui, ui.ui): baseui = ui.ui() - - from rhodecode.lib.utils import get_repos - repos = get_repos(repos_path) - - repos_list = {} - for name, path in repos: + for name, path in get_repos(repos_path): try: - #name = name.split('/')[-1] if repos_list.has_key(name): - raise RepositoryError('Duplicate repository name %s found in' - ' %s' % (name, path)) + raise RepositoryError('Duplicate repository name %s ' + 'found in %s' % (name, path)) else: + + klass = get_backend(path[0]) + if path[0] == 'hg': - repos_list[name] = MercurialRepository(path[1], baseui=baseui) - repos_list[name].name = name + repos_list[name] = klass(path[1], baseui=baseui) if path[0] == 'git': - repos_list[name] = GitRepository(path[1]) - repos_list[name].name = name - - dbrepo = None - if not initial: - #for initial scann on application first run we don't - #have db repos yet. - dbrepo = self.sa.query(Repository)\ - .options(joinedload(Repository.fork))\ - .filter(Repository.repo_name == name)\ - .scalar() - - if dbrepo: - log.info('Adding db instance to cached list') - repos_list[name].dbrepo = dbrepo - repos_list[name].description = dbrepo.description - if dbrepo.user: - repos_list[name].contact = dbrepo.user.full_contact - else: - repos_list[name].contact = self.sa.query(User)\ - .filter(User.admin == True).first().full_contact + repos_list[name] = klass(path[1]) except OSError: continue return repos_list - def get_repos(self): - for name, repo in _get_repos_cached().items(): + def get_repos(self, all_repos=None): + """ + Get all repos from db and for each such repo make backend and + fetch dependent data from db + """ + if not all_repos: + all_repos = self.sa.query(Repository).all() - if isinstance(repo, MercurialRepository) and repo._get_hidden(): - #skip hidden web repository - continue + for r in all_repos: + + repo = self.get(r.repo_name) - last_change = repo.last_change - tip = h.get_changeset_safe(repo, 'tip') + if repo is not None: + last_change = repo.last_change + tip = h.get_changeset_safe(repo, 'tip') - tmp_d = {} - tmp_d['name'] = repo.name - tmp_d['name_sort'] = tmp_d['name'].lower() - tmp_d['description'] = repo.description - tmp_d['description_sort'] = tmp_d['description'] - tmp_d['last_change'] = last_change - tmp_d['last_change_sort'] = time.mktime(last_change.timetuple()) - tmp_d['tip'] = tip.raw_id - tmp_d['tip_sort'] = tip.revision - tmp_d['rev'] = tip.revision - tmp_d['contact'] = repo.contact - tmp_d['contact_sort'] = tmp_d['contact'] - tmp_d['repo_archives'] = list(repo._get_archives()) - tmp_d['last_msg'] = tip.message - tmp_d['repo'] = repo - yield tmp_d + tmp_d = {} + tmp_d['name'] = repo.name + tmp_d['name_sort'] = tmp_d['name'].lower() + tmp_d['description'] = repo.dbrepo.description + tmp_d['description_sort'] = tmp_d['description'] + tmp_d['last_change'] = last_change + tmp_d['last_change_sort'] = time.mktime(last_change.timetuple()) + tmp_d['tip'] = tip.raw_id + tmp_d['tip_sort'] = tip.revision + tmp_d['rev'] = tip.revision + tmp_d['contact'] = repo.dbrepo.user.full_contact + tmp_d['contact_sort'] = tmp_d['contact'] + tmp_d['repo_archives'] = list(repo._get_archives()) + tmp_d['last_msg'] = tip.message + tmp_d['repo'] = repo + yield tmp_d def get_repo(self, repo_name): - try: - repo = _get_repos_cached()[repo_name] - return repo - except KeyError: - #i we're here and we got key errors let's try to invalidate the - #cahce and try again - invalidate_cache('cached_repo_list') - repo = _get_repos_cached()[repo_name] + return self.get(repo_name) + + def get(self, repo_name): + """ + Get's repository from given name, creates BackendInstance and + propagates it's data from database with all additional information + :param repo_name: + """ + if not HasRepoPermissionAny('repository.read', 'repository.write', + 'repository.admin')(repo_name, 'get repo check'): + return + + @cache_region('long_term', 'get_repo_cached_%s' % repo_name) + def _get_repo(repo_name): + + repo = vcs_get_repo(os.path.join(self.repos_path, repo_name), + alias=None, create=False) + + #skip hidden web repository + if isinstance(repo, MercurialRepository) and repo._get_hidden(): + return + + dbrepo = self.sa.query(Repository)\ + .options(joinedload(Repository.fork))\ + .options(joinedload(Repository.user))\ + .filter(Repository.repo_name == repo_name)\ + .scalar() + repo.dbrepo = dbrepo return repo + invalidate = False + if invalidate: + log.info('INVALIDATING CACHE FOR %s', repo_name) + region_invalidate(_get_repo, None, repo_name) + return _get_repo(repo_name)
--- a/rhodecode/templates/admin/users/user_edit_my_account.html Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/templates/admin/users/user_edit_my_account.html Sun Nov 07 15:02:56 2010 +0100 @@ -100,32 +100,32 @@ %for repo in c.user_repos: <tr> <td> - %if repo.dbrepo.repo_type =='hg': + %if repo['repo'].dbrepo.repo_type =='hg': <img class="icon" title="${_('Mercurial repository')}" alt="${_('Mercurial repository')}" src="/images/icons/hgicon.png"/> - %elif repo.dbrepo.repo_type =='git': + %elif repo['repo'].dbrepo.repo_type =='git': <img class="icon" title="${_('Git repository')}" alt="${_('Git repository')}" src="/images/icons/giticon.png"/> %else: %endif - %if repo.dbrepo.private: + %if repo['repo'].dbrepo.private: <img class="icon" alt="${_('private')}" src="/images/icons/lock.png"/> %else: <img class="icon" alt="${_('public')}" src="/images/icons/lock_open.png"/> %endif - ${h.link_to(repo.name, h.url('summary_home',repo_name=repo.name),class_="repo_name")} - %if repo.dbrepo.fork: - <a href="${h.url('summary_home',repo_name=repo.dbrepo.fork.repo_name)}"> + ${h.link_to(repo['repo'].name, h.url('summary_home',repo_name=repo['repo'].name),class_="repo_name")} + %if repo['repo'].dbrepo.fork: + <a href="${h.url('summary_home',repo_name=repo['repo'].dbrepo.fork.repo_name)}"> <img class="icon" alt="${_('public')}" - title="${_('Fork of')} ${repo.dbrepo.fork.repo_name}" + title="${_('Fork of')} ${repo['repo'].dbrepo.fork.repo_name}" src="/images/icons/arrow_divide.png"/></a> %endif </td> - <td><span class="tooltip" tooltip_title="${repo.last_change}">${("r%s:%s") % (h.get_changeset_safe(repo,'tip').revision,h.short_id(h.get_changeset_safe(repo,'tip').raw_id))}</span></td> - <td><a href="${h.url('repo_settings_home',repo_name=repo.name)}" title="${_('edit')}"><img class="icon" alt="${_('private')}" src="/images/icons/application_form_edit.png"/></a></td> + <td><span class="tooltip" tooltip_title="${repo['repo'].last_change}">${("r%s:%s") % (h.get_changeset_safe(repo['repo'],'tip').revision,h.short_id(h.get_changeset_safe(repo['repo'],'tip').raw_id))}</span></td> + <td><a href="${h.url('repo_settings_home',repo_name=repo['repo'].name)}" title="${_('edit')}"><img class="icon" alt="${_('private')}" src="/images/icons/application_form_edit.png"/></a></td> <td> - ${h.form(url('repo_settings_delete', repo_name=repo.name),method='delete')} - ${h.submit('remove_%s' % repo.name,'',class_="delete_icon action_button",onclick="return confirm('Confirm to delete this repository');")} + ${h.form(url('repo_settings_delete', repo_name=repo['repo'].name),method='delete')} + ${h.submit('remove_%s' % repo['repo'].name,'',class_="delete_icon action_button",onclick="return confirm('Confirm to delete this repository');")} ${h.end_form()} </td> </tr>
--- a/rhodecode/templates/base/base.html Sat Nov 06 16:19:01 2010 +0100 +++ b/rhodecode/templates/base/base.html Sun Nov 07 15:02:56 2010 +0100 @@ -98,11 +98,12 @@ <span>↓</span> </a> <ul class="repo_switcher"> - %for repo,private in c.repo_switcher_list: - %if private: - <li>${h.link_to(repo,h.url('summary_home',repo_name=repo),class_="private_repo")}</li> + %for repo in c.cached_repo_list: + + %if repo['repo'].dbrepo.private: + <li>${h.link_to(repo['repo'].name,h.url('summary_home',repo_name=repo['repo'].name),class_="private_repo %s" % repo['repo'].dbrepo.repo_type)}</li> %else: - <li>${h.link_to(repo,h.url('summary_home',repo_name=repo),class_="public_repo")}</li> + <li>${h.link_to(repo['repo'].name,h.url('summary_home',repo_name=repo['repo'].name),class_="public_repo %s" % repo['repo'].dbrepo.repo_type)}</li> %endif %endfor </ul>