# HG changeset patch # User Marcin Kuzminski # Date 1353668054 -3600 # Node ID 023f7873ef591169251973e2a72afbd20605a3dc # Parent eaa36a2497a9a23a8ee4bcc4f7d75c1e09296642 added caching layer into RSS/ATOM feeds - updated code for new LimitedDiffContainers - invalidate rss/atom cache keys on push - diff -r eaa36a2497a9 -r 023f7873ef59 rhodecode/controllers/feed.py --- a/rhodecode/controllers/feed.py Thu Nov 22 23:04:34 2012 +0100 +++ b/rhodecode/controllers/feed.py Fri Nov 23 11:54:14 2012 +0100 @@ -28,12 +28,14 @@ from pylons import url, response, tmpl_context as c from pylons.i18n.translation import _ +from beaker.cache import cache_region, region_invalidate from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed from rhodecode.lib import helpers as h from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from rhodecode.lib.base import BaseRepoController -from rhodecode.lib.diffs import DiffProcessor +from rhodecode.lib.diffs import DiffProcessor, LimitedDiffContainer +from rhodecode.model.db import CacheInvalidation log = logging.getLogger(__name__) @@ -51,6 +53,9 @@ self.language = 'en-us' self.ttl = "5" self.feed_nr = 20 + # we need to protect from parsing huge diffs here other way + # we can kill the server, 32*1024 chars is a reasonable limit + self.feed_diff_limit = 32 * 1024 def _get_title(self, cs): return "%s" % ( @@ -59,26 +64,28 @@ def __changes(self, cs): changes = [] - _diff = cs.diff() - # we need to protect from parsing huge diffs here other way - # we can kill the server, 32*1024 chars is a reasonable limit - HUGE_DIFF = 32 * 1024 - if len(_diff) > HUGE_DIFF: - changes = ['\n ' + _('Changeset was too big and was cut off...')] - return changes - diffprocessor = DiffProcessor(_diff) - stats = diffprocessor.prepare(inline_diff=False) - for st in stats: + diff_processor = DiffProcessor(cs.diff(), + diff_limit=self.feed_diff_limit) + _parsed = diff_processor.prepare(inline_diff=False) + limited_diff = False + if isinstance(_parsed, LimitedDiffContainer): + limited_diff = True + + for st in _parsed: st.update({'added': st['stats'][0], 'removed': st['stats'][1]}) changes.append('\n %(operation)s %(filename)s ' '(%(added)s lines added, %(removed)s lines removed)' % st) + if limited_diff: + changes = changes + ['\n ' + + _('Changeset was too big and was cut off...')] return changes def __get_desc(self, cs): desc_msg = [] - desc_msg.append('%s %s %s:
' % (cs.author, _('commited on'), + desc_msg.append('%s %s %s
' % (h.person(cs.author), + _('commited on'), h.fmt_date(cs.date))) #branches, tags, bookmarks if cs.branch: @@ -102,46 +109,67 @@ def atom(self, repo_name): """Produce an atom-1.0 feed via feedgenerator module""" - feed = Atom1Feed( - title=self.title % repo_name, - link=url('summary_home', repo_name=repo_name, - qualified=True), - description=self.description % repo_name, - language=self.language, - ttl=self.ttl - ) + + @cache_region('long_term') + def _get_feed_from_cache(key): + feed = Atom1Feed( + title=self.title % repo_name, + link=url('summary_home', repo_name=repo_name, + qualified=True), + description=self.description % repo_name, + language=self.language, + ttl=self.ttl + ) - for cs in reversed(list(c.rhodecode_repo[-self.feed_nr:])): - feed.add_item(title=self._get_title(cs), - link=url('changeset_home', repo_name=repo_name, - revision=cs.raw_id, qualified=True), - author_name=cs.author, - description=''.join(self.__get_desc(cs)), - pubdate=cs.date, - ) + for cs in reversed(list(c.rhodecode_repo[-self.feed_nr:])): + feed.add_item(title=self._get_title(cs), + link=url('changeset_home', repo_name=repo_name, + revision=cs.raw_id, qualified=True), + author_name=cs.author, + description=''.join(self.__get_desc(cs)), + pubdate=cs.date, + ) - response.content_type = feed.mime_type - return feed.writeString('utf-8') + response.content_type = feed.mime_type + return feed.writeString('utf-8') + + key = repo_name + '_ATOM' + inv = CacheInvalidation.invalidate(key) + if inv is not None: + region_invalidate(_get_feed_from_cache, None, key) + CacheInvalidation.set_valid(inv.cache_key) + return _get_feed_from_cache(key) def rss(self, repo_name): """Produce an rss2 feed via feedgenerator module""" - feed = Rss201rev2Feed( - title=self.title % repo_name, - link=url('summary_home', repo_name=repo_name, - qualified=True), - description=self.description % repo_name, - language=self.language, - ttl=self.ttl - ) + + @cache_region('long_term') + def _get_feed_from_cache(key): + feed = Rss201rev2Feed( + title=self.title % repo_name, + link=url('summary_home', repo_name=repo_name, + qualified=True), + description=self.description % repo_name, + language=self.language, + ttl=self.ttl + ) - for cs in reversed(list(c.rhodecode_repo[-self.feed_nr:])): - feed.add_item(title=self._get_title(cs), - link=url('changeset_home', repo_name=repo_name, - revision=cs.raw_id, qualified=True), - author_name=cs.author, - description=''.join(self.__get_desc(cs)), - pubdate=cs.date, - ) + for cs in reversed(list(c.rhodecode_repo[-self.feed_nr:])): + feed.add_item(title=self._get_title(cs), + link=url('changeset_home', repo_name=repo_name, + revision=cs.raw_id, qualified=True), + author_name=cs.author, + description=''.join(self.__get_desc(cs)), + pubdate=cs.date, + ) - response.content_type = feed.mime_type - return feed.writeString('utf-8') + response.content_type = feed.mime_type + return feed.writeString('utf-8') + + key = repo_name + '_RSS' + inv = CacheInvalidation.invalidate(key) + if inv is not None: + region_invalidate(_get_feed_from_cache, None, key) + CacheInvalidation.set_valid(inv.cache_key) + return _get_feed_from_cache(key) + diff -r eaa36a2497a9 -r 023f7873ef59 rhodecode/lib/utils2.py --- a/rhodecode/lib/utils2.py Thu Nov 22 23:04:34 2012 +0100 +++ b/rhodecode/lib/utils2.py Fri Nov 23 11:54:14 2012 +0100 @@ -279,6 +279,18 @@ return safe_str +def remove_suffix(s, suffix): + if s.endswith(suffix): + s = s[:-1 * len(suffix)] + return s + + +def remove_prefix(s, prefix): + if s.startswith(prefix): + s = s[:-1 * len(prefix)] + return s + + def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): """ Custom engine_from_config functions that makes sure we use NullPool for diff -r eaa36a2497a9 -r 023f7873ef59 rhodecode/model/db.py --- a/rhodecode/model/db.py Thu Nov 22 23:04:34 2012 +0100 +++ b/rhodecode/model/db.py Fri Nov 23 11:54:14 2012 +0100 @@ -46,7 +46,7 @@ from rhodecode.lib.vcs.utils.lazy import LazyProperty from rhodecode.lib.utils2 import str2bool, safe_str, get_changeset_safe, \ - safe_unicode + safe_unicode, remove_suffix from rhodecode.lib.compat import json from rhodecode.lib.caching_query import FromCache @@ -941,6 +941,7 @@ @LazyProperty def scm_instance(self): + return self.scm_instance_cached() return self.__get_instance() def scm_instance_cached(self, cache_map=None): @@ -1440,7 +1441,11 @@ iid = rhodecode.CONFIG.get('instance_id') if iid: prefix = iid - return "%s%s" % (prefix, key), prefix, key.rstrip('_README') + #remove specific suffixes like _README or _RSS + key = remove_suffix(key, '_README') + key = remove_suffix(key, '_RSS') + key = remove_suffix(key, '_ATOM') + return "%s%s" % (prefix, key), prefix, key @classmethod def get_by_key(cls, key):