Mercurial > kallithea
comparison rhodecode/model/scm.py @ 3960:5293d4bbb1ea
Merged dev into stable/default/master branch
author | Marcin Kuzminski <marcin@python-works.com> |
---|---|
date | Fri, 07 Jun 2013 00:31:11 +0200 |
parents | 51596d9ef2f8 ebde99f10d77 |
children | 3648a2b2e17a |
comparison
equal
deleted
inserted
replaced
3879:51596d9ef2f8 | 3960:5293d4bbb1ea |
---|---|
28 import time | 28 import time |
29 import traceback | 29 import traceback |
30 import logging | 30 import logging |
31 import cStringIO | 31 import cStringIO |
32 import pkg_resources | 32 import pkg_resources |
33 from os.path import dirname as dn, join as jn | 33 from os.path import join as jn |
34 | 34 |
35 from sqlalchemy import func | 35 from sqlalchemy import func |
36 from pylons.i18n.translation import _ | 36 from pylons.i18n.translation import _ |
37 | 37 |
38 import rhodecode | 38 import rhodecode |
44 | 44 |
45 from rhodecode import BACKENDS | 45 from rhodecode import BACKENDS |
46 from rhodecode.lib import helpers as h | 46 from rhodecode.lib import helpers as h |
47 from rhodecode.lib.utils2 import safe_str, safe_unicode, get_server_url,\ | 47 from rhodecode.lib.utils2 import safe_str, safe_unicode, get_server_url,\ |
48 _set_extras | 48 _set_extras |
49 from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny | 49 from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny,\ |
50 HasUserGroupPermissionAny | |
50 from rhodecode.lib.utils import get_filesystem_repos, make_ui, \ | 51 from rhodecode.lib.utils import get_filesystem_repos, make_ui, \ |
51 action_logger, REMOVED_REPO_PAT | 52 action_logger |
52 from rhodecode.model import BaseModel | 53 from rhodecode.model import BaseModel |
53 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \ | 54 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \ |
54 UserFollowing, UserLog, User, RepoGroup, PullRequest | 55 UserFollowing, UserLog, User, RepoGroup, PullRequest |
55 from rhodecode.lib.hooks import log_push_action | 56 from rhodecode.lib.hooks import log_push_action |
57 from rhodecode.lib.exceptions import NonRelativePathError | |
56 | 58 |
57 log = logging.getLogger(__name__) | 59 log = logging.getLogger(__name__) |
58 | 60 |
59 | 61 |
60 class UserTemp(object): | 62 class UserTemp(object): |
94 | 96 |
95 def __repr__(self): | 97 def __repr__(self): |
96 return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) | 98 return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) |
97 | 99 |
98 def __iter__(self): | 100 def __iter__(self): |
99 # pre-propagated cache_map to save executing select statements | 101 # pre-propagated valid_cache_keys to save executing select statements |
100 # for each repo | 102 # for each repo |
101 cache_map = CacheInvalidation.get_cache_map() | 103 valid_cache_keys = CacheInvalidation.get_valid_cache_keys() |
102 | 104 |
103 for dbr in self.db_repo_list: | 105 for dbr in self.db_repo_list: |
104 scmr = dbr.scm_instance_cached(cache_map) | 106 scmr = dbr.scm_instance_cached(valid_cache_keys) |
105 # check permission at this level | 107 # check permission at this level |
106 if not HasRepoPermissionAny( | 108 if not HasRepoPermissionAny( |
107 *self.perm_set | 109 *self.perm_set)(dbr.repo_name, 'get repo check'): |
108 )(dbr.repo_name, 'get repo check'): | |
109 continue | 110 continue |
110 | 111 |
111 try: | 112 try: |
112 last_change = scmr.last_change | 113 last_change = scmr.last_change |
113 tip = h.get_changeset_safe(scmr, 'tip') | 114 tip = h.get_changeset_safe(scmr, 'tip') |
148 | 149 |
149 def __iter__(self): | 150 def __iter__(self): |
150 for dbr in self.db_repo_list: | 151 for dbr in self.db_repo_list: |
151 # check permission at this level | 152 # check permission at this level |
152 if not HasRepoPermissionAny( | 153 if not HasRepoPermissionAny( |
153 *self.perm_set | 154 *self.perm_set)(dbr.repo_name, 'get repo check'): |
154 )(dbr.repo_name, 'get repo check'): | |
155 continue | 155 continue |
156 | 156 |
157 tmp_d = {} | 157 tmp_d = {} |
158 tmp_d['name'] = dbr.repo_name | 158 tmp_d['name'] = dbr.repo_name |
159 tmp_d['name_sort'] = tmp_d['name'].lower() | 159 tmp_d['name_sort'] = tmp_d['name'].lower() |
163 tmp_d['dbrepo'] = dbr.get_dict() | 163 tmp_d['dbrepo'] = dbr.get_dict() |
164 tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {} | 164 tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {} |
165 yield tmp_d | 165 yield tmp_d |
166 | 166 |
167 | 167 |
168 class GroupList(object): | 168 class _PermCheckIterator(object): |
169 def __init__(self, obj_list, obj_attr, perm_set, perm_checker): | |
170 """ | |
171 Creates iterator from given list of objects, additionally | |
172 checking permission for them from perm_set var | |
173 | |
174 :param obj_list: list of db objects | |
175 :param obj_attr: attribute of object to pass into perm_checker | |
176 :param perm_set: list of permissions to check | |
177 :param perm_checker: callable to check permissions against | |
178 """ | |
179 self.obj_list = obj_list | |
180 self.obj_attr = obj_attr | |
181 self.perm_set = perm_set | |
182 self.perm_checker = perm_checker | |
183 | |
184 def __len__(self): | |
185 return len(self.obj_list) | |
186 | |
187 def __repr__(self): | |
188 return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) | |
189 | |
190 def __iter__(self): | |
191 for db_obj in self.obj_list: | |
192 # check permission at this level | |
193 name = getattr(db_obj, self.obj_attr, None) | |
194 if not self.perm_checker(*self.perm_set)(name, self.__class__.__name__): | |
195 continue | |
196 | |
197 yield db_obj | |
198 | |
199 | |
200 class RepoList(_PermCheckIterator): | |
201 | |
202 def __init__(self, db_repo_list, perm_set=None): | |
203 if not perm_set: | |
204 perm_set = ['repository.read', 'repository.write', 'repository.admin'] | |
205 | |
206 super(RepoList, self).__init__(obj_list=db_repo_list, | |
207 obj_attr='repo_name', perm_set=perm_set, | |
208 perm_checker=HasRepoPermissionAny) | |
209 | |
210 | |
211 class RepoGroupList(_PermCheckIterator): | |
169 | 212 |
170 def __init__(self, db_repo_group_list, perm_set=None): | 213 def __init__(self, db_repo_group_list, perm_set=None): |
171 """ | |
172 Creates iterator from given list of group objects, additionally | |
173 checking permission for them from perm_set var | |
174 | |
175 :param db_repo_group_list: | |
176 :param perm_set: list of permissons to check | |
177 """ | |
178 self.db_repo_group_list = db_repo_group_list | |
179 if not perm_set: | 214 if not perm_set: |
180 perm_set = ['group.read', 'group.write', 'group.admin'] | 215 perm_set = ['group.read', 'group.write', 'group.admin'] |
181 self.perm_set = perm_set | 216 |
182 | 217 super(RepoGroupList, self).__init__(obj_list=db_repo_group_list, |
183 def __len__(self): | 218 obj_attr='group_name', perm_set=perm_set, |
184 return len(self.db_repo_group_list) | 219 perm_checker=HasReposGroupPermissionAny) |
185 | 220 |
186 def __repr__(self): | 221 |
187 return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) | 222 class UserGroupList(_PermCheckIterator): |
188 | 223 |
189 def __iter__(self): | 224 def __init__(self, db_user_group_list, perm_set=None): |
190 for dbgr in self.db_repo_group_list: | 225 if not perm_set: |
191 # check permission at this level | 226 perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin'] |
192 if not HasReposGroupPermissionAny( | 227 |
193 *self.perm_set | 228 super(UserGroupList, self).__init__(obj_list=db_user_group_list, |
194 )(dbgr.group_name, 'get group repo check'): | 229 obj_attr='users_group_name', perm_set=perm_set, |
195 continue | 230 perm_checker=HasUserGroupPermissionAny) |
196 | |
197 yield dbgr | |
198 | 231 |
199 | 232 |
200 class ScmModel(BaseModel): | 233 class ScmModel(BaseModel): |
201 """ | 234 """ |
202 Generic Scm Model | 235 Generic Scm Model |
291 | 324 |
292 def get_repos_groups(self, all_groups=None): | 325 def get_repos_groups(self, all_groups=None): |
293 if all_groups is None: | 326 if all_groups is None: |
294 all_groups = RepoGroup.query()\ | 327 all_groups = RepoGroup.query()\ |
295 .filter(RepoGroup.group_parent_id == None).all() | 328 .filter(RepoGroup.group_parent_id == None).all() |
296 return [x for x in GroupList(all_groups)] | 329 return [x for x in RepoGroupList(all_groups)] |
297 | 330 |
298 def mark_for_invalidation(self, repo_name): | 331 def mark_for_invalidation(self, repo_name): |
299 """ | 332 """ |
300 Puts cache invalidation task into db for | 333 Mark caches of this repo invalid in the database. |
301 further global cache invalidation | 334 |
302 | 335 :param repo_name: the repo for which caches should be marked invalid |
303 :param repo_name: this repo that should invalidation take place | 336 """ |
304 """ | 337 CacheInvalidation.set_invalidate(repo_name) |
305 invalidated_keys = CacheInvalidation.set_invalidate(repo_name=repo_name) | |
306 repo = Repository.get_by_repo_name(repo_name) | 338 repo = Repository.get_by_repo_name(repo_name) |
307 if repo: | 339 if repo: |
308 repo.update_changeset_cache() | 340 repo.update_changeset_cache() |
309 return invalidated_keys | |
310 | 341 |
311 def toggle_following_repo(self, follow_repo_id, user_id): | 342 def toggle_following_repo(self, follow_repo_id, user_id): |
312 | 343 |
313 f = self.sa.query(UserFollowing)\ | 344 f = self.sa.query(UserFollowing)\ |
314 .filter(UserFollowing.follows_repo_id == follow_repo_id)\ | 345 .filter(UserFollowing.follows_repo_id == follow_repo_id)\ |
453 Returns InMemoryCommit class based on scm_type | 484 Returns InMemoryCommit class based on scm_type |
454 | 485 |
455 :param scm_type: | 486 :param scm_type: |
456 """ | 487 """ |
457 if scm_type == 'hg': | 488 if scm_type == 'hg': |
458 from rhodecode.lib.vcs.backends.hg import \ | 489 from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset |
459 MercurialInMemoryChangeset as IMC | 490 return MercurialInMemoryChangeset |
460 elif scm_type == 'git': | 491 |
461 from rhodecode.lib.vcs.backends.git import \ | 492 if scm_type == 'git': |
462 GitInMemoryChangeset as IMC | 493 from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset |
463 return IMC | 494 return GitInMemoryChangeset |
495 | |
496 raise Exception('Invalid scm_type, must be one of hg,git got %s' | |
497 % (scm_type,)) | |
464 | 498 |
465 def pull_changes(self, repo, username): | 499 def pull_changes(self, repo, username): |
466 dbrepo = self.__get_repo(repo) | 500 dbrepo = self.__get_repo(repo) |
467 clone_uri = dbrepo.clone_uri | 501 clone_uri = dbrepo.clone_uri |
468 if not clone_uri: | 502 if not clone_uri: |
471 repo = dbrepo.scm_instance | 505 repo = dbrepo.scm_instance |
472 repo_name = dbrepo.repo_name | 506 repo_name = dbrepo.repo_name |
473 try: | 507 try: |
474 if repo.alias == 'git': | 508 if repo.alias == 'git': |
475 repo.fetch(clone_uri) | 509 repo.fetch(clone_uri) |
510 # git doesn't really have something like post-fetch action | |
511 # we fake that now. #TODO: extract fetched revisions somehow | |
512 # here | |
513 self._handle_push(repo, | |
514 username=username, | |
515 action='push_remote', | |
516 repo_name=repo_name, | |
517 revisions=[]) | |
476 else: | 518 else: |
477 self._handle_rc_scm_extras(username, dbrepo.repo_name, | 519 self._handle_rc_scm_extras(username, dbrepo.repo_name, |
478 repo.alias, action='push_remote') | 520 repo.alias, action='push_remote') |
479 repo.pull(clone_uri) | 521 repo.pull(clone_uri) |
480 | 522 |
514 action='push_local', | 556 action='push_local', |
515 repo_name=repo_name, | 557 repo_name=repo_name, |
516 revisions=[tip.raw_id]) | 558 revisions=[tip.raw_id]) |
517 return tip | 559 return tip |
518 | 560 |
519 def create_node(self, repo, repo_name, cs, user, author, message, content, | 561 def create_nodes(self, user, repo, message, nodes, parent_cs=None, |
520 f_path): | 562 author=None, trigger_push_hook=True): |
563 """ | |
564 Commits given multiple nodes into repo | |
565 | |
566 :param user: RhodeCode User object or user_id, the commiter | |
567 :param repo: RhodeCode Repository object | |
568 :param message: commit message | |
569 :param nodes: mapping {filename:{'content':content},...} | |
570 :param parent_cs: parent changeset, can be empty than it's initial commit | |
571 :param author: author of commit, cna be different that commiter only for git | |
572 :param trigger_push_hook: trigger push hooks | |
573 | |
574 :returns: new commited changeset | |
575 """ | |
576 | |
521 user = self._get_user(user) | 577 user = self._get_user(user) |
522 IMC = self._get_IMC_module(repo.alias) | 578 scm_instance = repo.scm_instance_no_cache() |
523 | 579 |
524 # decoding here will force that we have proper encoded values | 580 processed_nodes = [] |
525 # in any other case this will throw exceptions and deny commit | 581 for f_path in nodes: |
526 if isinstance(content, (basestring,)): | 582 if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path: |
527 content = safe_str(content) | 583 raise NonRelativePathError('%s is not an relative path' % f_path) |
528 elif isinstance(content, (file, cStringIO.OutputType,)): | 584 if f_path: |
529 content = content.read() | 585 f_path = os.path.normpath(f_path) |
530 else: | 586 content = nodes[f_path]['content'] |
531 raise Exception('Content is of unrecognized type %s' % ( | 587 f_path = safe_str(f_path) |
532 type(content) | 588 # decoding here will force that we have proper encoded values |
533 )) | 589 # in any other case this will throw exceptions and deny commit |
590 if isinstance(content, (basestring,)): | |
591 content = safe_str(content) | |
592 elif isinstance(content, (file, cStringIO.OutputType,)): | |
593 content = content.read() | |
594 else: | |
595 raise Exception('Content is of unrecognized type %s' % ( | |
596 type(content) | |
597 )) | |
598 processed_nodes.append((f_path, content)) | |
534 | 599 |
535 message = safe_unicode(message) | 600 message = safe_unicode(message) |
536 author = safe_unicode(author) | 601 commiter = user.full_contact |
537 path = safe_str(f_path) | 602 author = safe_unicode(author) if author else commiter |
538 m = IMC(repo) | 603 |
539 | 604 IMC = self._get_IMC_module(scm_instance.alias) |
540 if isinstance(cs, EmptyChangeset): | 605 imc = IMC(scm_instance) |
606 | |
607 if not parent_cs: | |
608 parent_cs = EmptyChangeset(alias=scm_instance.alias) | |
609 | |
610 if isinstance(parent_cs, EmptyChangeset): | |
541 # EmptyChangeset means we we're editing empty repository | 611 # EmptyChangeset means we we're editing empty repository |
542 parents = None | 612 parents = None |
543 else: | 613 else: |
544 parents = [cs] | 614 parents = [parent_cs] |
545 | 615 # add multiple nodes |
546 m.add(FileNode(path, content=content)) | 616 for path, content in processed_nodes: |
547 tip = m.commit(message=message, | 617 imc.add(FileNode(path, content=content)) |
548 author=author, | 618 |
549 parents=parents, branch=cs.branch) | 619 tip = imc.commit(message=message, |
550 | 620 author=author, |
551 self.mark_for_invalidation(repo_name) | 621 parents=parents, |
552 self._handle_push(repo, | 622 branch=parent_cs.branch) |
553 username=user.username, | 623 |
554 action='push_local', | 624 self.mark_for_invalidation(repo.repo_name) |
555 repo_name=repo_name, | 625 if trigger_push_hook: |
556 revisions=[tip.raw_id]) | 626 self._handle_push(scm_instance, |
627 username=user.username, | |
628 action='push_local', | |
629 repo_name=repo.repo_name, | |
630 revisions=[tip.raw_id]) | |
557 return tip | 631 return tip |
558 | 632 |
559 def get_nodes(self, repo_name, revision, root_path='/', flat=True): | 633 def get_nodes(self, repo_name, revision, root_path='/', flat=True): |
560 """ | 634 """ |
561 recursive walk in root dir and return a set of all path in that dir | 635 recursive walk in root dir and return a set of all path in that dir |
593 """ | 667 """ |
594 Generates select option with tags branches and bookmarks (for hg only) | 668 Generates select option with tags branches and bookmarks (for hg only) |
595 grouped by type | 669 grouped by type |
596 | 670 |
597 :param repo: | 671 :param repo: |
598 :type repo: | |
599 """ | 672 """ |
600 | 673 |
601 hist_l = [] | 674 hist_l = [] |
602 choices = [] | 675 choices = [] |
603 repo = self.__get_repo(repo) | 676 repo = self.__get_repo(repo) |
652 _rhodecode_hook = False | 725 _rhodecode_hook = False |
653 log.debug('Installing git hook in repo %s' % repo) | 726 log.debug('Installing git hook in repo %s' % repo) |
654 if os.path.exists(_hook_file): | 727 if os.path.exists(_hook_file): |
655 # let's take a look at this hook, maybe it's rhodecode ? | 728 # let's take a look at this hook, maybe it's rhodecode ? |
656 log.debug('hook exists, checking if it is from rhodecode') | 729 log.debug('hook exists, checking if it is from rhodecode') |
657 _HOOK_VER_PAT = re.compile(r'^RC_HOOK_VER') | |
658 with open(_hook_file, 'rb') as f: | 730 with open(_hook_file, 'rb') as f: |
659 data = f.read() | 731 data = f.read() |
660 matches = re.compile(r'(?:%s)\s*=\s*(.*)' | 732 matches = re.compile(r'(?:%s)\s*=\s*(.*)' |
661 % 'RC_HOOK_VER').search(data) | 733 % 'RC_HOOK_VER').search(data) |
662 if matches: | 734 if matches: |
669 else: | 741 else: |
670 # there is no hook in this dir, so we want to create one | 742 # there is no hook in this dir, so we want to create one |
671 _rhodecode_hook = True | 743 _rhodecode_hook = True |
672 | 744 |
673 if _rhodecode_hook or force_create: | 745 if _rhodecode_hook or force_create: |
674 log.debug('writing %s hook file !' % h_type) | 746 log.debug('writing %s hook file !' % (h_type,)) |
675 with open(_hook_file, 'wb') as f: | 747 with open(_hook_file, 'wb') as f: |
676 tmpl = tmpl.replace('_TMPL_', rhodecode.__version__) | 748 tmpl = tmpl.replace('_TMPL_', rhodecode.__version__) |
677 f.write(tmpl) | 749 f.write(tmpl) |
678 os.chmod(_hook_file, 0755) | 750 os.chmod(_hook_file, 0755) |
679 else: | 751 else: |