Mercurial > kallithea
changeset 692:cb0d9ce6ac5c beta
#50 on point cache invalidation changes.
Created cacheInvalidation table
cleaned up sa sessions from models, since it wasn't really needed.
| author | Marcin Kuzminski <marcin@python-works.com> |
|---|---|
| date | Mon, 15 Nov 2010 02:26:19 +0100 |
| parents | 7486da5f0628 |
| children | 90ac3255c117 |
| files | rhodecode/controllers/admin/permissions.py rhodecode/controllers/admin/repos.py rhodecode/controllers/admin/settings.py rhodecode/controllers/admin/users.py rhodecode/lib/auth.py rhodecode/lib/celerylib/tasks.py rhodecode/lib/utils.py rhodecode/model/db.py rhodecode/model/permission.py rhodecode/model/permission_model.py rhodecode/model/repo.py rhodecode/model/scm.py rhodecode/model/user.py |
| diffstat | 13 files changed, 223 insertions(+), 154 deletions(-) [+] |
line wrap: on
line diff
--- a/rhodecode/controllers/admin/permissions.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/controllers/admin/permissions.py Mon Nov 15 02:26:19 2010 +0100 @@ -31,7 +31,7 @@ from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator from rhodecode.lib.base import BaseController, render from rhodecode.model.forms import UserForm, DefaultPermissionsForm -from rhodecode.model.permission_model import PermissionModel +from rhodecode.model.permission import PermissionModel from rhodecode.model.user import UserModel import formencode import logging
--- a/rhodecode/controllers/admin/repos.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/controllers/admin/repos.py Mon Nov 15 02:26:19 2010 +0100 @@ -133,7 +133,7 @@ form_result = _form.to_python(dict(request.POST)) repo_model.update(repo_name, form_result) invalidate_cache('get_repo_cached_%s' % repo_name) - h.flash(_('Repository %s updated succesfully' % repo_name), + h.flash(_('Repository %s updated successfully' % repo_name), category='success') changed_name = form_result['repo_name'] action_logger(self.rhodecode_user, 'admin_updated_repo', @@ -152,7 +152,7 @@ except Exception: log.error(traceback.format_exc()) - h.flash(_('error occured during update of repository %s') \ + h.flash(_('error occurred during update of repository %s') \ % repo_name, category='error') return redirect(url('edit_repo', repo_name=changed_name))
--- a/rhodecode/controllers/admin/settings.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/controllers/admin/settings.py Mon Nov 15 02:26:19 2010 +0100 @@ -248,7 +248,7 @@ """ # url('admin_settings_my_account') - c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False) + c.user = UserModel().get(c.rhodecode_user.user_id, cache=False) all_repos = self.sa.query(Repository)\ .filter(Repository.user_id == c.user.user_id)\ .order_by(func.lower(Repository.repo_name))\ @@ -289,7 +289,7 @@ except formencode.Invalid, errors: c.user = user_model.get(c.rhodecode_user.user_id, cache=False) - c.user = UserModel(self.sa).get(c.rhodecode_user.user_id, cache=False) + c.user = UserModel().get(c.rhodecode_user.user_id, cache=False) all_repos = self.sa.query(Repository)\ .filter(Repository.user_id == c.user.user_id)\ .order_by(func.lower(Repository.repo_name))\
--- a/rhodecode/controllers/admin/users.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/controllers/admin/users.py Mon Nov 15 02:26:19 2010 +0100 @@ -17,7 +17,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. -from rhodecode.lib.utils import action_logger """ Created on April 4, 2010 users controller for pylons
--- a/rhodecode/lib/auth.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/lib/auth.py Mon Nov 15 02:26:19 2010 +0100 @@ -143,7 +143,7 @@ #=========================================================================== # fetch default permissions #=========================================================================== - default_user = UserModel(sa).get_by_username('default', cache=True) + default_user = UserModel().get_by_username('default', cache=True) default_perms = sa.query(RepoToPerm, Repository, Permission)\ .join((Repository, RepoToPerm.repository_id == Repository.repo_id))\
--- a/rhodecode/lib/celerylib/tasks.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/lib/celerylib/tasks.py Mon Nov 15 02:26:19 2010 +0100 @@ -1,16 +1,18 @@ from celery.decorators import task +import os +import traceback +from time import mktime + from operator import itemgetter from pylons.i18n.translation import _ from rhodecode.lib.celerylib import run_task, locked_task from rhodecode.lib.helpers import person from rhodecode.lib.smtp_mailer import SmtpMailer from rhodecode.lib.utils import OrderedDict -from time import mktime -import os -import traceback from vcs.backends import get_repo -from rhodecode.model.scm import ScmModel +from rhodecode.model.db import RhodeCodeUi + try: import json except ImportError: @@ -44,6 +46,11 @@ return sa +def get_repos_path(): + sa = get_session() + q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() + return q.ui_value + @task @locked_task def whoosh_index(repo_location, full_index): @@ -62,7 +69,7 @@ commits_by_day_author_aggregate = {} commits_by_day_aggregate = {} - repos_path = ScmModel().repos_path + repos_path = get_repos_path() p = os.path.join(repos_path, repo_name) repo = get_repo(p) @@ -271,10 +278,10 @@ from rhodecode.model.repo import RepoModel from vcs import get_backend log = create_repo_fork.get_logger() - repo_model = RepoModel(get_session()) + repo_model = RepoModel() repo_model.create(form_data, cur_user, just_db=True, fork=True) repo_name = form_data['repo_name'] - repos_path = ScmModel().repos_path + repos_path = get_repos_path() repo_path = os.path.join(repos_path, repo_name) repo_fork_path = os.path.join(repos_path, form_data['fork_name']) alias = form_data['repo_type'] @@ -291,7 +298,7 @@ 's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws'] - repos_path = ScmModel().repos_path + repos_path = get_repos_path() p = os.path.join(repos_path, repo_name) repo = get_repo(p) tip = repo.get_changeset()
--- a/rhodecode/lib/utils.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/lib/utils.py Mon Nov 15 02:26:19 2010 +0100 @@ -89,7 +89,7 @@ if hasattr(user, 'user_id'): user_obj = user elif isinstance(user, basestring): - user_obj = UserModel(sa).get_by_username(user, cache=False) + user_obj = UserModel().get_by_username(user, cache=False) else: raise Exception('You have to provide user object or username') @@ -97,7 +97,7 @@ if repo: repo_name = repo.lstrip('/') - repository = RepoModel(sa).get(repo_name, cache=False) + repository = RepoModel().get(repo_name, cache=False) if not repository: raise Exception('You have to provide valid repository') else: @@ -293,12 +293,16 @@ for k, v in hgsettings.items(): config[k] = v -def invalidate_cache(name, *args): +def invalidate_cache(cache_key, *args): """ Puts cache invalidation task into db for further global cache invalidation """ - pass + from rhodecode.model.scm import ScmModel + + if cache_key.startswith('get_repo_cached_'): + name = cache_key.split('get_repo_cached_')[-1] + ScmModel().mark_for_invalidation(name) class EmptyChangeset(BaseChangeset): """ @@ -340,7 +344,7 @@ """ sa = meta.Session() - rm = RepoModel(sa) + rm = RepoModel() user = sa.query(User).filter(User.admin == True).first() for name, repo in initial_repo_list.items():
--- a/rhodecode/model/db.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/model/db.py Mon Nov 15 02:26:19 2010 +0100 @@ -90,9 +90,8 @@ repo_to_perm = relation('RepoToPerm', cascade='all') stats = relation('Statistics', cascade='all') - def __repr__(self): - return "<Repository('id:%s:%s')>" % (self.repo_id, self.repo_name) + return "<Repository('%s:%s')>" % (self.repo_id, self.repo_name) class Permission(Base): __tablename__ = 'permissions' @@ -140,10 +139,17 @@ class CacheInvalidation(Base): __tablename__ = 'cache_invalidation' - __table_args__ = {'useexisting':True} + __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True}) cache_id = Column("cache_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True) cache_key = Column("cache_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) cache_args = Column("cache_args", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) - cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=None) + cache_active = Column("cache_active", BOOLEAN(), nullable=True, unique=None, default=False) + def __init__(self, cache_key, cache_args=''): + self.cache_key = cache_key + self.cache_args = cache_args + self.cache_active = False + + def __repr__(self): + return "<CacheInvaidation('%s:%s')>" % (self.cache_id, self.cache_key)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rhodecode/model/permission.py Mon Nov 15 02:26:19 2010 +0100 @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Model for permissions +# Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License or (at your opinion) any later version of the license. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +""" +Created on Aug 20, 2010 +Model for permissions +@author: marcink +""" + +from rhodecode.model.db import User, Permission, UserToPerm, RepoToPerm +from rhodecode.model.caching_query import FromCache +from rhodecode.model.meta import Session +import logging +import traceback +log = logging.getLogger(__name__) + + +class PermissionModel(object): + + def __init__(self): + self.sa = Session() + + def get_permission(self, permission_id, cache=False): + perm = self.sa.query(Permission) + if cache: + perm = perm.options(FromCache("sql_cache_short", + "get_permission_%s" % permission_id)) + return perm.get(permission_id) + + def get_permission_by_name(self, name, cache=False): + perm = self.sa.query(Permission)\ + .filter(Permission.permission_name == name) + if cache: + perm = perm.options(FromCache("sql_cache_short", + "get_permission_%s" % name)) + return perm.scalar() + + def update(self, form_result): + perm_user = self.sa.query(User)\ + .filter(User.username == form_result['perm_user_name']).scalar() + u2p = self.sa.query(UserToPerm).filter(UserToPerm.user == perm_user).all() + if len(u2p) != 3: + raise Exception('Defined: %s should be 3 permissions for default' + ' user. This should not happen please verify' + ' your database' % len(u2p)) + + try: + #stage 1 change defaults + for p in u2p: + if p.permission.permission_name.startswith('repository.'): + p.permission = self.get_permission_by_name( + form_result['default_perm']) + self.sa.add(p) + + if p.permission.permission_name.startswith('hg.register.'): + p.permission = self.get_permission_by_name( + form_result['default_register']) + self.sa.add(p) + + if p.permission.permission_name.startswith('hg.create.'): + p.permission = self.get_permission_by_name( + form_result['default_create']) + self.sa.add(p) + #stage 2 update all default permissions for repos if checked + if form_result['overwrite_default'] == 'true': + for r2p in self.sa.query(RepoToPerm)\ + .filter(RepoToPerm.user == perm_user).all(): + r2p.permission = self.get_permission_by_name( + form_result['default_perm']) + self.sa.add(r2p) + + #stage 3 set anonymous access + if perm_user.username == 'default': + perm_user.active = bool(form_result['anonymous']) + self.sa.add(perm_user) + + + self.sa.commit() + except: + log.error(traceback.format_exc()) + self.sa.rollback() + raise + + + + +
--- a/rhodecode/model/permission_model.py Sun Nov 14 22:54:16 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,106 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -# Model for permissions -# Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; version 2 -# of the License or (at your opinion) any later version of the license. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, -# MA 02110-1301, USA. -""" -Created on Aug 20, 2010 -Model for permissions -@author: marcink -""" - -from rhodecode.model.db import User, Permission, UserToPerm, RepoToPerm -from rhodecode.model.caching_query import FromCache -from rhodecode.model.meta import Session -import logging -import traceback -log = logging.getLogger(__name__) - - -class PermissionModel(object): - - def __init__(self, sa=None): - if not sa: - self.sa = Session() - else: - self.sa = sa - - def get_permission(self, permission_id, cache=False): - perm = self.sa.query(Permission) - if cache: - perm = perm.options(FromCache("sql_cache_short", - "get_permission_%s" % permission_id)) - return perm.get(permission_id) - - def get_permission_by_name(self, name, cache=False): - perm = self.sa.query(Permission)\ - .filter(Permission.permission_name == name) - if cache: - perm = perm.options(FromCache("sql_cache_short", - "get_permission_%s" % name)) - return perm.scalar() - - def update(self, form_result): - perm_user = self.sa.query(User)\ - .filter(User.username == form_result['perm_user_name']).scalar() - u2p = self.sa.query(UserToPerm).filter(UserToPerm.user == perm_user).all() - if len(u2p) != 3: - raise Exception('Defined: %s should be 3 permissions for default' - ' user. This should not happen please verify' - ' your database' % len(u2p)) - - try: - #stage 1 change defaults - for p in u2p: - if p.permission.permission_name.startswith('repository.'): - p.permission = self.get_permission_by_name( - form_result['default_perm']) - self.sa.add(p) - - if p.permission.permission_name.startswith('hg.register.'): - p.permission = self.get_permission_by_name( - form_result['default_register']) - self.sa.add(p) - - if p.permission.permission_name.startswith('hg.create.'): - p.permission = self.get_permission_by_name( - form_result['default_create']) - self.sa.add(p) - #stage 2 update all default permissions for repos if checked - if form_result['overwrite_default'] == 'true': - for r2p in self.sa.query(RepoToPerm)\ - .filter(RepoToPerm.user == perm_user).all(): - r2p.permission = self.get_permission_by_name( - form_result['default_perm']) - self.sa.add(r2p) - - #stage 3 set anonymous access - if perm_user.username == 'default': - perm_user.active = bool(form_result['anonymous']) - self.sa.add(perm_user) - - - self.sa.commit() - except: - log.error(traceback.format_exc()) - self.sa.rollback() - raise - - - - -
--- a/rhodecode/model/repo.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/model/repo.py Mon Nov 15 02:26:19 2010 +0100 @@ -36,11 +36,8 @@ class RepoModel(object): - def __init__(self, sa=None): - if not sa: - self.sa = Session() - else: - self.sa = sa + def __init__(self): + self.sa = Session() def get(self, repo_id, cache=False): repo = self.sa.query(Repository)\ @@ -67,7 +64,7 @@ #update permissions for username, perm in form_data['perms_updates']: r2p = self.sa.query(RepoToPerm)\ - .filter(RepoToPerm.user == UserModel(self.sa).get_by_username(username, cache=False))\ + .filter(RepoToPerm.user == UserModel().get_by_username(username, cache=False))\ .filter(RepoToPerm.repository == self.get(repo_name))\ .one() @@ -80,7 +77,7 @@ for username, perm in form_data['perms_new']: r2p = RepoToPerm() r2p.repository = self.get(repo_name) - r2p.user = UserModel(self.sa).get_by_username(username, cache=False) + r2p.user = UserModel().get_by_username(username, cache=False) r2p.permission_id = self.sa.query(Permission).filter( Permission.permission_name == perm)\ @@ -134,7 +131,7 @@ #create default permission repo_to_perm = RepoToPerm() default = 'repository.read' - for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms: + for p in UserModel().get_by_username('default', cache=False).user_perms: if p.permission.permission_name.startswith('repository.'): default = p.permission.permission_name break @@ -146,7 +143,7 @@ .one().permission_id repo_to_perm.repository_id = new_repo.repo_id - repo_to_perm.user_id = UserModel(self.sa).get_by_username('default', cache=False).user_id + repo_to_perm.user_id = UserModel().get_by_username('default', cache=False).user_id self.sa.add(repo_to_perm) self.sa.commit()
--- a/rhodecode/model/scm.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/model/scm.py Mon Nov 15 02:26:19 2010 +0100 @@ -28,12 +28,15 @@ from rhodecode.lib.auth import HasRepoPermissionAny from rhodecode.lib.utils import get_repos from rhodecode.model import meta -from rhodecode.model.db import Repository, User, RhodeCodeUi +from rhodecode.model.db import Repository, User, RhodeCodeUi, CacheInvalidation +from rhodecode.model.caching_query import FromCache from sqlalchemy.orm import joinedload +from sqlalchemy.orm.session import make_transient from vcs import get_backend from vcs.utils.helpers import get_scm from vcs.exceptions import RepositoryError, VCSError from vcs.utils.lazy import LazyProperty +import traceback import logging import os import time @@ -45,12 +48,8 @@ Mercurial Model """ - def __init__(self, sa=None): - if not sa: - self.sa = meta.Session() - else: - self.sa = sa - + def __init__(self): + self.sa = meta.Session() @LazyProperty def repos_path(self): @@ -143,7 +142,7 @@ 'repository.admin')(repo_name, 'get repo check'): return - @cache_region('long_term', 'get_repo_cached_%s' % repo_name) + @cache_region('long_term') def _get_repo(repo_name): repo_path = os.path.join(self.repos_path, repo_name) @@ -165,13 +164,76 @@ .options(joinedload(Repository.user))\ .filter(Repository.repo_name == repo_name)\ .scalar() + make_transient(dbrepo) repo.dbrepo = dbrepo return repo - invalidate = False + invalidate = self._should_invalidate(repo_name) if invalidate: - log.info('INVALIDATING CACHE FOR %s', repo_name) + log.info('invalidating cache for repository %s', repo_name) region_invalidate(_get_repo, None, repo_name) + self._mark_invalidated(invalidate) return _get_repo(repo_name) + + + def mark_for_invalidation(self, repo_name): + """ + Puts cache invalidation task into db for + further global cache invalidation + + :param repo_name: this repo that should invalidation take place + """ + log.debug('marking %s for invalidation', repo_name) + cache = self.sa.query(CacheInvalidation)\ + .filter(CacheInvalidation.cache_key == repo_name).scalar() + + if cache: + #mark this cache as inactive + cache.cache_active = False + else: + log.debug('cache key not found in invalidation db -> creating one') + cache = CacheInvalidation(repo_name) + + try: + self.sa.add(cache) + self.sa.commit() + except: + log.error(traceback.format_exc()) + self.sa.rollback() + + + + + + def _should_invalidate(self, repo_name): + """ + Looks up database for invalidation signals for this repo_name + :param repo_name: + """ + + ret = self.sa.query(CacheInvalidation)\ + .options(FromCache('sql_cache_short', + 'get_invalidation_%s' % repo_name))\ + .filter(CacheInvalidation.cache_key == repo_name)\ + .filter(CacheInvalidation.cache_active == False)\ + .scalar() + + return ret + + def _mark_invalidated(self, cache_key): + """ + Marks all occurences of cache to invaldation as already invalidated + @param repo_name: + """ + if cache_key: + log.debug('marking %s as already invalidated', cache_key) + try: + cache_key.cache_active = True + self.sa.add(cache_key) + self.sa.commit() + except: + log.error(traceback.format_exc()) + self.sa.rollback() +
--- a/rhodecode/model/user.py Sun Nov 14 22:54:16 2010 +0100 +++ b/rhodecode/model/user.py Mon Nov 15 02:26:19 2010 +0100 @@ -36,11 +36,8 @@ class UserModel(object): - def __init__(self, sa=None): - if not sa: - self.sa = Session() - else: - self.sa = sa + def __init__(self): + self.sa = Session() def get(self, user_id, cache=False): user = self.sa.query(User)