Mercurial > kallithea

comparison rhodecode/model/scm.py @ 1038:5554aa9c2480 beta

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
another major code rafactor, reimplemented (almost from scratch) the way caching works, Should be solid rock for now. Some code optymizations on scmModel.get() to make it don't load unneded things. Changed db cache to file that should also reduce memory size
author Marcin Kuzminski <marcin@python-works.com>
date Sun, 13 Feb 2011 00:29:31 +0100
parents e2ebbb27df4e
children 51b70569c330
comparison
equal deleted inserted replaced
1037:b1d6478d4561 1038:5554aa9c2480
29 import traceback 29 import traceback
30 import logging 30 import logging
31 31
32 from mercurial import ui 32 from mercurial import ui
33 33
34 from sqlalchemy.orm import joinedload
35 from sqlalchemy.orm.session import make_transient
36 from sqlalchemy.exc import DatabaseError 34 from sqlalchemy.exc import DatabaseError
37 35
38 from beaker.cache import cache_region, region_invalidate 36 from beaker.cache import cache_region, region_invalidate
39 37
40 from vcs import get_backend 38 from vcs import get_backend
43 from vcs.utils.lazy import LazyProperty 41 from vcs.utils.lazy import LazyProperty
44 42
45 from rhodecode import BACKENDS 43 from rhodecode import BACKENDS
46 from rhodecode.lib import helpers as h 44 from rhodecode.lib import helpers as h
47 from rhodecode.lib.auth import HasRepoPermissionAny 45 from rhodecode.lib.auth import HasRepoPermissionAny
48 from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, action_logger 46 from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, \
47 action_logger
49 from rhodecode.model import BaseModel 48 from rhodecode.model import BaseModel
50 from rhodecode.model.user import UserModel 49 from rhodecode.model.user import UserModel
50 from rhodecode.model.repo import RepoModel
51 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \ 51 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
52 UserFollowing, UserLog 52 UserFollowing, UserLog
53 from rhodecode.model.caching_query import FromCache 53 from rhodecode.model.caching_query import FromCache
54 54
55 log = logging.getLogger(__name__) 55 log = logging.getLogger(__name__)
80 80
81 q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() 81 q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
82 82
83 return q.ui_value 83 return q.ui_value
84 84
85 def repo_scan(self, repos_path, baseui): 85 def repo_scan(self, repos_path=None):
86 """Listing of repositories in given path. This path should not be a 86 """Listing of repositories in given path. This path should not be a
87 repository itself. Return a dictionary of repository objects 87 repository itself. Return a dictionary of repository objects
88 88
89 :param repos_path: path to directory containing repositories 89 :param repos_path: path to directory containing repositories
90 :param baseui: baseui instance to instantiate MercurialRepostitory with
91 """ 90 """
92 91
93 log.info('scanning for repositories in %s', repos_path) 92 log.info('scanning for repositories in %s', repos_path)
94 93
95 if not isinstance(baseui, ui.ui): 94 if repos_path is None:
96 baseui = make_ui('db') 95 repos_path = self.repos_path
96
97 baseui = make_ui('db')
97 repos_list = {} 98 repos_list = {}
98 99
99 for name, path in get_filesystem_repos(repos_path, recursive=True): 100 for name, path in get_filesystem_repos(repos_path, recursive=True):
100 try: 101 try:
101 if repos_list.has_key(name): 102 if repos_list.has_key(name):
132 .filter(CacheInvalidation.cache_active == False)\ 133 .filter(CacheInvalidation.cache_active == False)\
133 .all()] 134 .all()]
134 135
135 for r in all_repos: 136 for r in all_repos:
136 137
137 repo = self.get(r.repo_name, invalidation_list) 138 repo, dbrepo = self.get(r.repo_name, invalidation_list)
138 139
139 if repo is not None: 140 if repo is not None:
140 last_change = repo.last_change 141 last_change = repo.last_change
141 tip = h.get_changeset_safe(repo, 'tip') 142 tip = h.get_changeset_safe(repo, 'tip')
142 143
143 tmp_d = {} 144 tmp_d = {}
144 tmp_d['name'] = r.repo_name 145 tmp_d['name'] = r.repo_name
145 tmp_d['name_sort'] = tmp_d['name'].lower() 146 tmp_d['name_sort'] = tmp_d['name'].lower()
146 tmp_d['description'] = repo.dbrepo.description 147 tmp_d['description'] = dbrepo.description
147 tmp_d['description_sort'] = tmp_d['description'] 148 tmp_d['description_sort'] = tmp_d['description']
148 tmp_d['last_change'] = last_change 149 tmp_d['last_change'] = last_change
149 tmp_d['last_change_sort'] = time.mktime(last_change.timetuple()) 150 tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
150 tmp_d['tip'] = tip.raw_id 151 tmp_d['tip'] = tip.raw_id
151 tmp_d['tip_sort'] = tip.revision 152 tmp_d['tip_sort'] = tip.revision
152 tmp_d['rev'] = tip.revision 153 tmp_d['rev'] = tip.revision
153 tmp_d['contact'] = repo.dbrepo.user.full_contact 154 tmp_d['contact'] = dbrepo.user.full_contact
154 tmp_d['contact_sort'] = tmp_d['contact'] 155 tmp_d['contact_sort'] = tmp_d['contact']
155 tmp_d['owner_sort'] = tmp_d['contact'] 156 tmp_d['owner_sort'] = tmp_d['contact']
156 tmp_d['repo_archives'] = list(repo._get_archives()) 157 tmp_d['repo_archives'] = list(repo._get_archives())
157 tmp_d['last_msg'] = tip.message 158 tmp_d['last_msg'] = tip.message
158 tmp_d['repo'] = repo 159 tmp_d['repo'] = repo
160 tmp_d['dbrepo'] = dbrepo
159 yield tmp_d 161 yield tmp_d
160 162
161 def get_repo(self, repo_name): 163 def get(self, repo_name, invalidation_list=None, retval='all'):
162 return self.get(repo_name) 164 """Returns a tuple of Repository,DbRepository,
163 165 Get's repository from given name, creates BackendInstance and
164 def get(self, repo_name, invalidation_list=None):
165 """Get's repository from given name, creates BackendInstance and
166 propagates it's data from database with all additional information 166 propagates it's data from database with all additional information
167 167
168 :param repo_name: 168 :param repo_name:
169 :param invalidation_list: if a invalidation list is given the get 169 :param invalidation_list: if a invalidation list is given the get
170 method should not manually check if this repository needs 170 method should not manually check if this repository needs
171 invalidation and just invalidate the repositories in list 171 invalidation and just invalidate the repositories in list
172 172 :param retval: string specifing what to return one of 'repo','dbrepo',
173 'all'if repo or dbrepo is given it'll just lazy load chosen type
174 and return None as the second
173 """ 175 """
174 if not HasRepoPermissionAny('repository.read', 'repository.write', 176 if not HasRepoPermissionAny('repository.read', 'repository.write',
175 'repository.admin')(repo_name, 'get repo check'): 177 'repository.admin')(repo_name, 'get repo check'):
176 return 178 return
177 179
187 alias = get_scm(repo_path)[0] 189 alias = get_scm(repo_path)[0]
188 log.debug('Creating instance of %s repository', alias) 190 log.debug('Creating instance of %s repository', alias)
189 backend = get_backend(alias) 191 backend = get_backend(alias)
190 except VCSError: 192 except VCSError:
191 log.error(traceback.format_exc()) 193 log.error(traceback.format_exc())
192 log.error('Perhaps this repository is in db and not in filesystem' 194 log.error('Perhaps this repository is in db and not in '
193 'run rescan repositories with "destroy old data "' 195 'filesystem run rescan repositories with '
194 'option from admin panel') 196 '"destroy old data " option from admin panel')
195 return 197 return
196 198
197 if alias == 'hg': 199 if alias == 'hg':
198 from pylons import app_globals as g 200 repo = backend(repo_path, create=False, baseui=make_ui('db'))
199 repo = backend(repo_path, create=False, baseui=g.baseui)
200 #skip hidden web repository 201 #skip hidden web repository
201 if repo._get_hidden(): 202 if repo._get_hidden():
202 return 203 return
203 else: 204 else:
204 repo = backend(repo_path, create=False) 205 repo = backend(repo_path, create=False)
205 206
206 dbrepo = self.sa.query(Repository)\
207 .options(joinedload(Repository.fork))\
208 .options(joinedload(Repository.user))\
209 .filter(Repository.repo_name == repo_name)\
210 .scalar()
211
212 self.sa.expunge_all()
213 log.debug('making transient %s', dbrepo)
214 make_transient(dbrepo)
215
216 for attr in ['user', 'forks', 'followers', 'group', 'repo_to_perm',
217 'users_group_to_perm', 'stats', 'logs']:
218 attr = getattr(dbrepo, attr, False)
219 if attr:
220 if isinstance(attr, list):
221 for a in attr:
222 log.debug('making transient %s', a)
223 make_transient(a)
224 else:
225 log.debug('making transient %s', attr)
226 make_transient(attr)
227
228 repo.dbrepo = dbrepo
229 return repo 207 return repo
230 208
231 pre_invalidate = True 209 pre_invalidate = True
210 dbinvalidate = False
211
232 if invalidation_list is not None: 212 if invalidation_list is not None:
233 pre_invalidate = repo_name in invalidation_list 213 pre_invalidate = repo_name in invalidation_list
234 214
235 if pre_invalidate: 215 if pre_invalidate:
216 #this returns object to invalidate
236 invalidate = self._should_invalidate(repo_name) 217 invalidate = self._should_invalidate(repo_name)
237
238 if invalidate: 218 if invalidate:
239 log.info('invalidating cache for repository %s', repo_name) 219 log.info('invalidating cache for repository %s', repo_name)
240 region_invalidate(_get_repo, None, repo_name) 220 #region_invalidate(_get_repo, None, repo_name)
241 self._mark_invalidated(invalidate) 221 self._mark_invalidated(invalidate)
242 222 dbinvalidate = True
243 return _get_repo(repo_name) 223
224 r, dbr = None, None
225 if retval == 'repo' or 'all':
226 r = _get_repo(repo_name)
227 if retval == 'dbrepo' or 'all':
228 dbr = RepoModel(self.sa).get_full(repo_name, cache=True,
229 invalidate=dbinvalidate)
230
231
232 return r, dbr
244 233
245 234
246 235
247 def mark_for_invalidation(self, repo_name): 236 def mark_for_invalidation(self, repo_name):
248 """Puts cache invalidation task into db for 237 """Puts cache invalidation task into db for
368 357
369 :param repo_name: 358 :param repo_name:
370 """ 359 """
371 360
372 ret = self.sa.query(CacheInvalidation)\ 361 ret = self.sa.query(CacheInvalidation)\
373 .options(FromCache('sql_cache_short',
374 'get_invalidation_%s' % repo_name))\
375 .filter(CacheInvalidation.cache_key == repo_name)\ 362 .filter(CacheInvalidation.cache_key == repo_name)\
376 .filter(CacheInvalidation.cache_active == False)\ 363 .filter(CacheInvalidation.cache_active == False)\
377 .scalar() 364 .scalar()
378 365
379 return ret 366 return ret
380 367
381 def _mark_invalidated(self, cache_key): 368 def _mark_invalidated(self, cache_key):
382 """ Marks all occurences of cache to invaldation as already invalidated 369 """ Marks all occurrences of cache to invalidation as already
370 invalidated
383 371
384 :param cache_key: 372 :param cache_key:
385 """ 373 """
386 374
387 if cache_key: 375 if cache_key: