Mercurial > kallithea
changeset 1607:e886f91fcb71 beta
Cached readme generation
- cleaned up cache system
| author | Marcin Kuzminski <marcin@python-works.com> |
|---|---|
| date | Fri, 28 Oct 2011 02:40:22 +0200 |
| parents | a30689fc4f61 |
| children | 0069657aaf42 |
| files | development.ini production.ini rhodecode/config/deployment.ini_tmpl rhodecode/controllers/summary.py rhodecode/model/db.py rhodecode/model/scm.py test.ini |
| diffstat | 7 files changed, 123 insertions(+), 87 deletions(-) [+] |
line wrap: on
line diff
--- a/development.ini Thu Oct 27 19:05:37 2011 +0200 +++ b/development.ini Fri Oct 28 02:40:22 2011 +0200 @@ -91,21 +91,27 @@ beaker.cache.super_short_term.type=memory beaker.cache.super_short_term.expire=10 +beaker.cache.super_short_term.key_length = 256 beaker.cache.short_term.type=memory beaker.cache.short_term.expire=60 +beaker.cache.short_term.key_length = 256 beaker.cache.long_term.type=memory beaker.cache.long_term.expire=36000 +beaker.cache.long_term.key_length = 256 beaker.cache.sql_cache_short.type=memory beaker.cache.sql_cache_short.expire=10 +beaker.cache.sql_cache_short.key_length = 256 beaker.cache.sql_cache_med.type=memory beaker.cache.sql_cache_med.expire=360 +beaker.cache.sql_cache_med.key_length = 256 beaker.cache.sql_cache_long.type=file beaker.cache.sql_cache_long.expire=3600 +beaker.cache.sql_cache_long.key_length = 256 #################################### ### BEAKER SESSION ####
--- a/production.ini Thu Oct 27 19:05:37 2011 +0200 +++ b/production.ini Fri Oct 28 02:40:22 2011 +0200 @@ -91,21 +91,27 @@ beaker.cache.super_short_term.type=memory beaker.cache.super_short_term.expire=10 +beaker.cache.super_short_term.key_length = 256 beaker.cache.short_term.type=memory beaker.cache.short_term.expire=60 +beaker.cache.short_term.key_length = 256 beaker.cache.long_term.type=memory beaker.cache.long_term.expire=36000 +beaker.cache.long_term.key_length = 256 beaker.cache.sql_cache_short.type=memory beaker.cache.sql_cache_short.expire=10 +beaker.cache.sql_cache_short.key_length = 256 beaker.cache.sql_cache_med.type=memory beaker.cache.sql_cache_med.expire=360 +beaker.cache.sql_cache_med.key_length = 256 beaker.cache.sql_cache_long.type=file beaker.cache.sql_cache_long.expire=3600 +beaker.cache.sql_cache_long.key_length = 256 #################################### ### BEAKER SESSION ####
--- a/rhodecode/config/deployment.ini_tmpl Thu Oct 27 19:05:37 2011 +0200 +++ b/rhodecode/config/deployment.ini_tmpl Fri Oct 28 02:40:22 2011 +0200 @@ -91,21 +91,27 @@ beaker.cache.super_short_term.type=memory beaker.cache.super_short_term.expire=10 +beaker.cache.super_short_term.key_length = 256 beaker.cache.short_term.type=memory beaker.cache.short_term.expire=60 +beaker.cache.short_term.key_length = 256 beaker.cache.long_term.type=memory beaker.cache.long_term.expire=36000 +beaker.cache.long_term.key_length = 256 beaker.cache.sql_cache_short.type=memory beaker.cache.sql_cache_short.expire=10 +beaker.cache.sql_cache_short.key_length = 256 beaker.cache.sql_cache_med.type=memory beaker.cache.sql_cache_med.expire=360 +beaker.cache.sql_cache_med.key_length = 256 beaker.cache.sql_cache_long.type=file beaker.cache.sql_cache_long.expire=3600 +beaker.cache.sql_cache_long.key_length = 256 #################################### ### BEAKER SESSION #### @@ -154,6 +160,7 @@ # MySQL # sqlalchemy.db1.url = mysql://user:pass@localhost/rhodecode +# see sqlalchemy docs for others sqlalchemy.db1.echo = false sqlalchemy.db1.pool_recycle = 3600
--- a/rhodecode/controllers/summary.py Thu Oct 27 19:05:37 2011 +0200 +++ b/rhodecode/controllers/summary.py Fri Oct 28 02:40:22 2011 +0200 @@ -36,7 +36,9 @@ from pylons import tmpl_context as c, request, url from pylons.i18n.translation import _ -from rhodecode.model.db import Statistics +from beaker.cache import cache_region, region_invalidate + +from rhodecode.model.db import Statistics, CacheInvalidation from rhodecode.lib import ALL_READMES, ALL_EXTS from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseRepoController, render @@ -50,7 +52,7 @@ log = logging.getLogger(__name__) -README_FILES = [''.join([x[0][0], x[1][0]]) for x in +README_FILES = [''.join([x[0][0], x[1][0]]) for x in sorted(list(product(ALL_READMES, ALL_EXTS)), key=lambda y:y[0][1] + y[1][1])] @@ -166,32 +168,43 @@ if c.enable_downloads: c.download_options = self._get_download_links(c.rhodecode_repo) - c.readme_data,c.readme_file = self.__get_readme_data() + c.readme_data, c.readme_file = self.__get_readme_data(c.rhodecode_repo) return render('summary/summary.html') - def __get_readme_data(self): - readme_data = None - readme_file = None - - try: - cs = c.rhodecode_repo.get_changeset('tip') - renderer = MarkupRenderer() - for f in README_FILES: - try: - readme = cs.get_node(f) - readme_file = f - readme_data = renderer.render(readme.content, f) - break - except NodeDoesNotExistError: - continue - except ChangesetError: - pass - except EmptyRepositoryError: - pass - except Exception: - log.error(traceback.format_exc()) + def __get_readme_data(self, repo): - return readme_data, readme_file + @cache_region('long_term') + def _get_readme_from_cache(key): + readme_data = None + readme_file = None + log.debug('Fetching readme file') + try: + cs = repo.get_changeset('tip') + renderer = MarkupRenderer() + for f in README_FILES: + try: + readme = cs.get_node(f) + readme_file = f + readme_data = renderer.render(readme.content, f) + log.debug('Found readme %s' % readme_file) + break + except NodeDoesNotExistError: + continue + except ChangesetError: + pass + except EmptyRepositoryError: + pass + except Exception: + log.error(traceback.format_exc()) + + return readme_data, readme_file + + key = repo.name + '_README' + inv = CacheInvalidation.invalidate(key) + if inv is not None: + region_invalidate(_get_readme_from_cache, None, key) + CacheInvalidation.set_valid(inv.cache_key) + return _get_readme_from_cache(key) def _get_download_links(self, repo):
--- a/rhodecode/model/db.py Thu Oct 27 19:05:37 2011 +0200 +++ b/rhodecode/model/db.py Fri Oct 28 02:40:22 2011 +0200 @@ -663,29 +663,13 @@ @property def invalidate(self): - """ - Returns Invalidation object if this repo should be invalidated - None otherwise. `cache_active = False` means that this cache - state is not valid and needs to be invalidated - """ - return CacheInvalidation.query()\ - .filter(CacheInvalidation.cache_key == self.repo_name)\ - .filter(CacheInvalidation.cache_active == False)\ - .scalar() + return CacheInvalidation.invalidate(self.repo_name) def set_invalidate(self): """ set a cache for invalidation for this instance """ - inv = CacheInvalidation.query()\ - .filter(CacheInvalidation.cache_key == self.repo_name)\ - .scalar() - - if inv is None: - inv = CacheInvalidation(self.repo_name) - inv.cache_active = True - Session.add(inv) - Session.commit() + CacheInvalidation.set_invalidate(self.repo_name) @LazyProperty def scm_instance(self): @@ -696,19 +680,13 @@ @cache_region('long_term') def _c(repo_name): return self.__get_instance() - - # TODO: remove this trick when beaker 1.6 is released - # and have fixed this issue with not supporting unicode keys - rn = safe_str(self.repo_name) + rn = self.repo_name inv = self.invalidate if inv is not None: region_invalidate(_c, None, rn) # update our cache - inv.cache_active = True - Session.add(inv) - Session.commit() - + CacheInvalidation.set_valid(inv.cache_key) return _c(rn) def __get_instance(self): @@ -730,7 +708,7 @@ repo = backend(safe_str(repo_full_path), create=False, baseui=self._ui) - #skip hidden web repository + # skip hidden web repository if repo._get_hidden(): return else: @@ -855,7 +833,7 @@ :param group_name: """ - path_prefix = (self.parent_group.full_path_splitted if + path_prefix = (self.parent_group.full_path_splitted if self.parent_group else []) return Group.url_sep().join(path_prefix + [group_name]) @@ -1060,6 +1038,57 @@ return "<%s('%s:%s')>" % (self.__class__.__name__, self.cache_id, self.cache_key) + @classmethod + def invalidate(cls, key): + """ + Returns Invalidation object if this given key should be invalidated + None otherwise. `cache_active = False` means that this cache + state is not valid and needs to be invalidated + + :param key: + """ + return cls.query()\ + .filter(CacheInvalidation.cache_key == key)\ + .filter(CacheInvalidation.cache_active == False)\ + .scalar() + + @classmethod + def set_invalidate(cls, key): + """ + Mark this Cache key for invalidation + + :param key: + """ + + log.debug('marking %s for invalidation' % key) + inv_obj = Session().query(cls)\ + .filter(cls.cache_key == key).scalar() + if inv_obj: + inv_obj.cache_active = False + else: + log.debug('cache key not found in invalidation db -> creating one') + inv_obj = CacheInvalidation(key) + + try: + Session.add(inv_obj) + Session.commit() + except Exception: + log.error(traceback.format_exc()) + Session.rollback() + + @classmethod + def set_valid(cls, key): + """ + Mark this cache key as active and currently cached + + :param key: + """ + inv_obj = Session().query(CacheInvalidation)\ + .filter(CacheInvalidation.cache_key == key).scalar() + inv_obj.cache_active = True + Session.add(inv_obj) + Session.commit() + class DbMigrateVersion(Base, BaseModel): __tablename__ = 'db_migrate_version' __table_args__ = {'extend_existing':True}
--- a/rhodecode/model/scm.py Thu Oct 27 19:05:37 2011 +0200 +++ b/rhodecode/model/scm.py Fri Oct 28 02:40:22 2011 +0200 @@ -197,24 +197,8 @@ :param repo_name: this repo that should invalidation take place """ - - log.debug('marking %s for invalidation', repo_name) - cache = self.sa.query(CacheInvalidation)\ - .filter(CacheInvalidation.cache_key == repo_name).scalar() - - if cache: - # mark this cache as inactive - cache.cache_active = False - else: - log.debug('cache key not found in invalidation db -> creating one') - cache = CacheInvalidation(repo_name) - - try: - self.sa.add(cache) - self.sa.commit() - except (DatabaseError,): - log.error(traceback.format_exc()) - self.sa.rollback() + CacheInvalidation.set_invalidate(repo_name) + CacheInvalidation.set_invalidate(repo_name+"_README") def toggle_following_repo(self, follow_repo_id, user_id): @@ -395,20 +379,5 @@ self.mark_for_invalidation(repo_name) - def get_unread_journal(self): return self.sa.query(UserLog).count() - - def _should_invalidate(self, repo_name): - """Looks up database for invalidation signals for this repo_name - - :param repo_name: - """ - - ret = self.sa.query(CacheInvalidation)\ - .filter(CacheInvalidation.cache_key == repo_name)\ - .filter(CacheInvalidation.cache_active == False)\ - .scalar() - - return ret -
--- a/test.ini Thu Oct 27 19:05:37 2011 +0200 +++ b/test.ini Fri Oct 28 02:40:22 2011 +0200 @@ -88,21 +88,27 @@ beaker.cache.super_short_term.type=memory beaker.cache.super_short_term.expire=10 +beaker.cache.super_short_term.key_length = 256 beaker.cache.short_term.type=memory beaker.cache.short_term.expire=60 +beaker.cache.short_term.key_length = 256 beaker.cache.long_term.type=memory beaker.cache.long_term.expire=36000 +beaker.cache.long_term.key_length = 256 beaker.cache.sql_cache_short.type=memory beaker.cache.sql_cache_short.expire=10 +beaker.cache.sql_cache_short.key_length = 256 beaker.cache.sql_cache_med.type=memory beaker.cache.sql_cache_med.expire=360 +beaker.cache.sql_cache_med.key_length = 256 beaker.cache.sql_cache_long.type=file beaker.cache.sql_cache_long.expire=3600 +beaker.cache.sql_cache_long.key_length = 256 #################################### ### BEAKER SESSION #### @@ -143,7 +149,7 @@ ######################################################### sqlalchemy.db1.url = sqlite:///%(here)s/test.db #sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode_tests -#sqlalchemy.db1.echo = False +#sqlalchemy.db1.echo = false #sqlalchemy.db1.pool_recycle = 3600 sqlalchemy.convert_unicode = true