Mercurial > kallithea
changeset 343:6484963056cd
implemented cache for repeated queries in simplehg mercurial requests
| author | Marcin Kuzminski <marcin@python-works.com> |
|---|---|
| date | Wed, 14 Jul 2010 12:31:11 +0200 |
| parents | c71dc6ef36e6 |
| children | ab5d91709af9 |
| files | development.ini production.ini pylons_app/lib/auth.py pylons_app/lib/middleware/simplehg.py pylons_app/lib/utils.py |
| diffstat | 5 files changed, 109 insertions(+), 90 deletions(-) [+] |
line wrap: on
line diff
--- a/development.ini Wed Jul 14 02:28:51 2010 +0200 +++ b/development.ini Wed Jul 14 12:31:11 2010 +0200 @@ -44,11 +44,13 @@ #################################### beaker.cache.data_dir=/%(here)s/data/cache/data beaker.cache.lock_dir=/%(here)s/data/cache/lock -beaker.cache.regions=short_term,long_term +beaker.cache.regions=super_short_term,short_term,long_term beaker.cache.long_term.type=memory beaker.cache.long_term.expire=36000 beaker.cache.short_term.type=memory beaker.cache.short_term.expire=60 +beaker.cache.super_short_term.type=memory +beaker.cache.super_short_term.expire=10 ################################################################################ ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ##
--- a/production.ini Wed Jul 14 02:28:51 2010 +0200 +++ b/production.ini Wed Jul 14 12:31:11 2010 +0200 @@ -39,17 +39,18 @@ lang=en cache_dir = %(here)s/data - #################################### ### BEAKER CACHE #### #################################### beaker.cache.data_dir=/%(here)s/data/cache/data beaker.cache.lock_dir=/%(here)s/data/cache/lock -beaker.cache.regions=short_term,long_term +beaker.cache.regions=super_short_term,short_term,long_term beaker.cache.long_term.type=memory beaker.cache.long_term.expire=36000 beaker.cache.short_term.type=memory beaker.cache.short_term.expire=60 +beaker.cache.super_short_term.type=memory +beaker.cache.super_short_term.expire=10 ################################################################################ ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ##
--- a/pylons_app/lib/auth.py Wed Jul 14 02:28:51 2010 +0200 +++ b/pylons_app/lib/auth.py Wed Jul 14 12:31:11 2010 +0200 @@ -22,18 +22,18 @@ @author: marcink """ - +from beaker.cache import cache_region from functools import wraps -from pylons import session, url, request +from pylons import config, session, url, request from pylons.controllers.util import abort, redirect +from pylons_app.lib.utils import get_repo_slug from pylons_app.model import meta from pylons_app.model.db import User, Repo2Perm, Repository, Permission -from pylons_app.lib.utils import get_repo_slug from sqlalchemy.exc import OperationalError from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound import crypt import logging -from pylons import config + log = logging.getLogger(__name__) def get_crypt_password(password): @@ -43,11 +43,17 @@ """ return crypt.crypt(password, '6a') + +@cache_region('super_short_term', 'cached_user') +def get_user_cached(username): + sa = meta.Session + user = sa.query(User).filter(User.username == username).one() + return user + def authfunc(environ, username, password): - sa = meta.Session password_crypt = get_crypt_password(password) try: - user = sa.query(User).filter(User.username == username).one() + user = get_user_cached(username) except (NoResultFound, MultipleResultsFound, OperationalError) as e: log.error(e) user = None
--- a/pylons_app/lib/middleware/simplehg.py Wed Jul 14 02:28:51 2010 +0200 +++ b/pylons_app/lib/middleware/simplehg.py Wed Jul 14 12:31:11 2010 +0200 @@ -2,7 +2,7 @@ # encoding: utf-8 # middleware to handle mercurial api calls # Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> - + # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; version 2 @@ -27,21 +27,23 @@ """ from datetime import datetime from itertools import chain +from mercurial.error import RepoError from mercurial.hgweb import hgweb from mercurial.hgweb.request import wsgiapplication -from mercurial.error import RepoError from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import REMOTE_USER, AUTH_TYPE -from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware +from pylons_app.lib.auth import authfunc, HasPermissionAnyMiddleware, \ + get_user_cached from pylons_app.lib.utils import is_mercurial, make_ui, invalidate_cache, \ check_repo_fast from pylons_app.model import meta from pylons_app.model.db import UserLog, User -import pylons_app.lib.helpers as h from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError import logging import os +import pylons_app.lib.helpers as h import traceback + log = logging.getLogger(__name__) class SimpleHg(object): @@ -56,86 +58,84 @@ def __call__(self, environ, start_response): if not is_mercurial(environ): return self.application(environ, start_response) - else: - #=================================================================== - # AUTHENTICATE THIS MERCURIAL REQUEST - #=================================================================== - username = REMOTE_USER(environ) - if not username: - result = self.authenticate(environ) - if isinstance(result, str): - AUTH_TYPE.update(environ, 'basic') - REMOTE_USER.update(environ, result) - else: - return result.wsgi_application(environ, start_response) - + + #=================================================================== + # AUTHENTICATE THIS MERCURIAL REQUEST + #=================================================================== + username = REMOTE_USER(environ) + if not username: + result = self.authenticate(environ) + if isinstance(result, str): + AUTH_TYPE.update(environ, 'basic') + REMOTE_USER.update(environ, result) + else: + return result.wsgi_application(environ, start_response) + + try: + repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:]) + except: + log.error(traceback.format_exc()) + return HTTPInternalServerError()(environ, start_response) + + #=================================================================== + # CHECK PERMISSIONS FOR THIS REQUEST + #=================================================================== + action = self.__get_action(environ) + if action: + username = self.__get_environ_user(environ) try: - repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:]) + user = self.__get_user(username) except: log.error(traceback.format_exc()) return HTTPInternalServerError()(environ, start_response) + #check permissions for this repository + if action == 'pull': + if not HasPermissionAnyMiddleware('repository.read', + 'repository.write', + 'repository.admin')\ + (user, repo_name): + return HTTPForbidden()(environ, start_response) + if action == 'push': + if not HasPermissionAnyMiddleware('repository.write', + 'repository.admin')\ + (user, repo_name): + return HTTPForbidden()(environ, start_response) - #=================================================================== - # CHECK PERMISSIONS FOR THIS REQUEST - #=================================================================== - action = self.__get_action(environ) - if action: - username = self.__get_environ_user(environ) - try: - sa = meta.Session - user = sa.query(User)\ - .filter(User.username == username).one() - except: - log.error(traceback.format_exc()) - return HTTPInternalServerError()(environ, start_response) - #check permissions for this repository - if action == 'pull': - if not HasPermissionAnyMiddleware('repository.read', - 'repository.write', - 'repository.admin')\ - (user, repo_name): - return HTTPForbidden()(environ, start_response) - if action == 'push': - if not HasPermissionAnyMiddleware('repository.write', - 'repository.admin')\ - (user, repo_name): - return HTTPForbidden()(environ, start_response) - - #log action - proxy_key = 'HTTP_X_REAL_IP' - def_key = 'REMOTE_ADDR' - ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0')) - self.__log_user_action(user, action, repo_name, ipaddr) - - #=================================================================== - # MERCURIAL REQUEST HANDLING - #=================================================================== - environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path - self.baseui = make_ui('db') - self.basepath = self.config['base_path'] - self.repo_path = os.path.join(self.basepath, repo_name) + #log action + proxy_key = 'HTTP_X_REAL_IP' + def_key = 'REMOTE_ADDR' + ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0')) + self.__log_user_action(user, action, repo_name, ipaddr) + + #=================================================================== + # MERCURIAL REQUEST HANDLING + #=================================================================== + environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path + self.baseui = make_ui('db') + self.basepath = self.config['base_path'] + self.repo_path = os.path.join(self.basepath, repo_name) - #quick check if that dir exists... - if check_repo_fast(repo_name, self.basepath): + #quick check if that dir exists... + if check_repo_fast(repo_name, self.basepath): + return HTTPNotFound()(environ, start_response) + try: + app = wsgiapplication(self.__make_app) + except RepoError as e: + if str(e).find('not found') != -1: return HTTPNotFound()(environ, start_response) - try: - app = wsgiapplication(self.__make_app) - except RepoError as e: - if str(e).find('not found') != -1: - return HTTPNotFound()(environ, start_response) - except Exception: - log.error(traceback.format_exc()) - return HTTPInternalServerError()(environ, start_response) - - #invalidate cache on push - if action == 'push': - self.__invalidate_cache(repo_name) - messages = [] - messages.append('thank you for using hg-app') - - return self.msg_wrapper(app, environ, start_response, messages) - else: - return app(environ, start_response) + except Exception: + log.error(traceback.format_exc()) + return HTTPInternalServerError()(environ, start_response) + + #invalidate cache on push + if action == 'push': + self.__invalidate_cache(repo_name) + messages = [] + messages.append('thank you for using hg-app') + + return self.msg_wrapper(app, environ, start_response, messages) + else: + return app(environ, start_response) def msg_wrapper(self, app, environ, start_response, messages=[]): @@ -160,6 +160,11 @@ def __get_environ_user(self, environ): return environ.get('REMOTE_USER') + def __get_user(self, username): + return get_user_cached(username) + + + def __get_size(self, repo_path, content_size): size = int(content_size) for path, dirs, files in os.walk(repo_path):
--- a/pylons_app/lib/utils.py Wed Jul 14 02:28:51 2010 +0200 +++ b/pylons_app/lib/utils.py Wed Jul 14 12:31:11 2010 +0200 @@ -16,6 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. +from beaker.cache import cache_region """ Created on April 18, 2010 @@ -75,6 +76,13 @@ log.info('%s repo is free for creation', repo_name) return True + +@cache_region('super_short_term', 'cached_hg_ui') +def get_hg_ui_cached(): + from pylons_app.model.meta import Session + sa = Session() + return sa.query(HgAppUi).all() + def make_ui(read_from='file', path=None, checkpaths=True): """ A function that will read python rc files or database @@ -112,10 +120,7 @@ elif read_from == 'db': - from pylons_app.model.meta import Session - sa = Session() - - hg_ui = sa.query(HgAppUi).all() + hg_ui = get_hg_ui_cached() for ui_ in hg_ui: baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)