Mercurial > kallithea

comparison rhodecode/lib/utils.py @ 2031:82a88013a3fd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merge 1.3 into stable
author Marcin Kuzminski <marcin@python-works.com>
date Sun, 26 Feb 2012 17:25:09 +0200
parents 6b318706f712 3a014a84a2db
children 9ab21c5ddb84
comparison
equal deleted inserted replaced
2005:ab0e122b38a7 2031:82a88013a3fd
5 5
6 Utilities library for RhodeCode 6 Utilities library for RhodeCode
7 7
8 :created_on: Apr 18, 2010 8 :created_on: Apr 18, 2010
9 :author: marcink 9 :author: marcink
10 :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com> 10 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
11 :license: GPLv3, see COPYING for more details. 11 :license: GPLv3, see COPYING for more details.
12 """ 12 """
13 # This program is free software: you can redistribute it and/or modify 13 # This program is free software: you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by 14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation, either version 3 of the License, or 15 # the Free Software Foundation, either version 3 of the License, or
27 import logging 27 import logging
28 import datetime 28 import datetime
29 import traceback 29 import traceback
30 import paste 30 import paste
31 import beaker 31 import beaker
32 import tarfile
33 import shutil
34 from os.path import abspath
32 from os.path import dirname as dn, join as jn 35 from os.path import dirname as dn, join as jn
33 36
34 from paste.script.command import Command, BadCommand 37 from paste.script.command import Command, BadCommand
35 38
36 from mercurial import ui, config 39 from mercurial import ui, config
37 40
38 from webhelpers.text import collapse, remove_formatting, strip_tags 41 from webhelpers.text import collapse, remove_formatting, strip_tags
39 42
40 from vcs import get_backend 43 from rhodecode.lib.vcs import get_backend
41 from vcs.backends.base import BaseChangeset 44 from rhodecode.lib.vcs.backends.base import BaseChangeset
42 from vcs.utils.lazy import LazyProperty 45 from rhodecode.lib.vcs.utils.lazy import LazyProperty
43 from vcs.utils.helpers import get_scm 46 from rhodecode.lib.vcs.utils.helpers import get_scm
44 from vcs.exceptions import VCSError 47 from rhodecode.lib.vcs.exceptions import VCSError
48
49 from rhodecode.lib.caching_query import FromCache
45 50
46 from rhodecode.model import meta 51 from rhodecode.model import meta
47 from rhodecode.model.caching_query import FromCache 52 from rhodecode.model.db import Repository, User, RhodeCodeUi, \
48 from rhodecode.model.db import Repository, User, RhodeCodeUi, UserLog, Group, \ 53 UserLog, RepoGroup, RhodeCodeSetting, UserRepoGroupToPerm
49 RhodeCodeSettings 54 from rhodecode.model.meta import Session
50 from rhodecode.model.repo import RepoModel 55 from rhodecode.model.repos_group import ReposGroupModel
51 56
52 log = logging.getLogger(__name__) 57 log = logging.getLogger(__name__)
53 58
54 59
55 def recursive_replace(str, replace=' '): 60 def recursive_replace(str_, replace=' '):
56 """Recursive replace of given sign to just one instance 61 """Recursive replace of given sign to just one instance
57 62
58 :param str: given string 63 :param str_: given string
59 :param replace: char to find and replace multiple instances 64 :param replace: char to find and replace multiple instances
60 65
61 Examples:: 66 Examples::
62 >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-') 67 >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
63 'Mighty-Mighty-Bo-sstones' 68 'Mighty-Mighty-Bo-sstones'
64 """ 69 """
65 70
66 if str.find(replace * 2) == -1: 71 if str_.find(replace * 2) == -1:
67 return str 72 return str_
68 else: 73 else:
69 str = str.replace(replace * 2, replace) 74 str_ = str_.replace(replace * 2, replace)
70 return recursive_replace(str, replace) 75 return recursive_replace(str_, replace)
71 76
72 77
73 def repo_name_slug(value): 78 def repo_name_slug(value):
74 """Return slug of name of repository 79 """Return slug of name of repository
75 This function is called on each creation/modification 80 This function is called on each creation/modification
88 93
89 def get_repo_slug(request): 94 def get_repo_slug(request):
90 return request.environ['pylons.routes_dict'].get('repo_name') 95 return request.environ['pylons.routes_dict'].get('repo_name')
91 96
92 97
93 def action_logger(user, action, repo, ipaddr='', sa=None): 98 def get_repos_group_slug(request):
99 return request.environ['pylons.routes_dict'].get('group_name')
100
101
102 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
94 """ 103 """
95 Action logger for various actions made by users 104 Action logger for various actions made by users
96 105
97 :param user: user that made this action, can be a unique username string or 106 :param user: user that made this action, can be a unique username string or
98 object containing user_id attribute 107 object containing user_id attribute
104 :param sa: optional sqlalchemy session 113 :param sa: optional sqlalchemy session
105 114
106 """ 115 """
107 116
108 if not sa: 117 if not sa:
109 sa = meta.Session() 118 sa = meta.Session
110 119
111 try: 120 try:
112 if hasattr(user, 'user_id'): 121 if hasattr(user, 'user_id'):
113 user_obj = user 122 user_obj = user
114 elif isinstance(user, basestring): 123 elif isinstance(user, basestring):
115 user_obj = User.get_by_username(user) 124 user_obj = User.get_by_username(user)
116 else: 125 else:
117 raise Exception('You have to provide user object or username') 126 raise Exception('You have to provide user object or username')
118 127
119 rm = RepoModel()
120 if hasattr(repo, 'repo_id'): 128 if hasattr(repo, 'repo_id'):
121 repo_obj = rm.get(repo.repo_id, cache=False) 129 repo_obj = Repository.get(repo.repo_id)
122 repo_name = repo_obj.repo_name 130 repo_name = repo_obj.repo_name
123 elif isinstance(repo, basestring): 131 elif isinstance(repo, basestring):
124 repo_name = repo.lstrip('/') 132 repo_name = repo.lstrip('/')
125 repo_obj = rm.get_by_repo_name(repo_name, cache=False) 133 repo_obj = Repository.get_by_repo_name(repo_name)
126 else: 134 else:
127 raise Exception('You have to provide repository to action logger') 135 raise Exception('You have to provide repository to action logger')
128 136
129 user_log = UserLog() 137 user_log = UserLog()
130 user_log.user_id = user_obj.user_id 138 user_log.user_id = user_obj.user_id
134 user_log.repository_name = repo_name 142 user_log.repository_name = repo_name
135 143
136 user_log.action_date = datetime.datetime.now() 144 user_log.action_date = datetime.datetime.now()
137 user_log.user_ip = ipaddr 145 user_log.user_ip = ipaddr
138 sa.add(user_log) 146 sa.add(user_log)
139 sa.commit() 147
140 148 log.info('Adding user %s, action %s on %s' % (user_obj, action, repo))
141 log.info('Adding user %s, action %s on %s', user_obj, action, repo) 149 if commit:
150 sa.commit()
142 except: 151 except:
143 log.error(traceback.format_exc()) 152 log.error(traceback.format_exc())
144 sa.rollback() 153 raise
145 154
146 155
147 def get_repos(path, recursive=False): 156 def get_repos(path, recursive=False):
148 """ 157 """
149 Scans given path for repos and return (name,(type,path)) tuple 158 Scans given path for repos and return (name,(type,path)) tuple
150 159
151 :param path: path to scann for repositories 160 :param path: path to scan for repositories
152 :param recursive: recursive search and return names with subdirs in front 161 :param recursive: recursive search and return names with subdirs in front
153 """ 162 """
154 from vcs.utils.helpers import get_scm
155 from vcs.exceptions import VCSError
156 163
157 # remove ending slash for better results 164 # remove ending slash for better results
158 path = path.rstrip('/') 165 path = path.rstrip(os.sep)
159 166
160 def _get_repos(p): 167 def _get_repos(p):
161 if not os.access(p, os.W_OK): 168 if not os.access(p, os.W_OK):
162 return 169 return
163 for dirpath in os.listdir(p): 170 for dirpath in os.listdir(p):
193 get_scm(full_path) 200 get_scm(full_path)
194 return True 201 return True
195 except VCSError: 202 except VCSError:
196 return False 203 return False
197 204
205
198 def is_valid_repos_group(repos_group_name, base_path): 206 def is_valid_repos_group(repos_group_name, base_path):
199 """ 207 """
200 Returns True if given path is a repos group False otherwise 208 Returns True if given path is a repos group False otherwise
201 209
202 :param repo_name: 210 :param repo_name:
203 :param base_path: 211 :param base_path:
204 """ 212 """
205 full_path = os.path.join(base_path, repos_group_name) 213 full_path = os.path.join(base_path, repos_group_name)
206 214
211 # check if it's a valid path 219 # check if it's a valid path
212 if os.path.isdir(full_path): 220 if os.path.isdir(full_path):
213 return True 221 return True
214 222
215 return False 223 return False
224
216 225
217 def ask_ok(prompt, retries=4, complaint='Yes or no, please!'): 226 def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
218 while True: 227 while True:
219 ok = raw_input(prompt) 228 ok = raw_input(prompt)
220 if ok in ('y', 'ye', 'yes'): 229 if ok in ('y', 'ye', 'yes'):
248 :param read_from: read from 'file' or 'db' 257 :param read_from: read from 'file' or 'db'
249 """ 258 """
250 259
251 baseui = ui.ui() 260 baseui = ui.ui()
252 261
253 #clean the baseui object 262 # clean the baseui object
254 baseui._ocfg = config.config() 263 baseui._ocfg = config.config()
255 baseui._ucfg = config.config() 264 baseui._ucfg = config.config()
256 baseui._tcfg = config.config() 265 baseui._tcfg = config.config()
257 266
258 if read_from == 'file': 267 if read_from == 'file':
259 if not os.path.isfile(path): 268 if not os.path.isfile(path):
260 log.warning('Unable to read config file %s' % path) 269 log.debug('hgrc file is not present at %s skipping...' % path)
261 return False 270 return False
262 log.debug('reading hgrc from %s', path) 271 log.debug('reading hgrc from %s' % path)
263 cfg = config.config() 272 cfg = config.config()
264 cfg.read(path) 273 cfg.read(path)
265 for section in ui_sections: 274 for section in ui_sections:
266 for k, v in cfg.items(section): 275 for k, v in cfg.items(section):
267 log.debug('settings ui from file[%s]%s:%s', section, k, v) 276 log.debug('settings ui from file[%s]%s:%s' % (section, k, v))
268 baseui.setconfig(section, k, v) 277 baseui.setconfig(section, k, v)
269 278
270 elif read_from == 'db': 279 elif read_from == 'db':
271 sa = meta.Session() 280 sa = meta.Session
272 ret = sa.query(RhodeCodeUi)\ 281 ret = sa.query(RhodeCodeUi)\
273 .options(FromCache("sql_cache_short", 282 .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\
274 "get_hg_ui_settings")).all() 283 .all()
275 284
276 hg_ui = ret 285 hg_ui = ret
277 for ui_ in hg_ui: 286 for ui_ in hg_ui:
278 if ui_.ui_active: 287 if ui_.ui_active:
279 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section, 288 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,
283 meta.Session.remove() 292 meta.Session.remove()
284 return baseui 293 return baseui
285 294
286 295
287 def set_rhodecode_config(config): 296 def set_rhodecode_config(config):
288 """Updates pylons config with new settings from database 297 """
298 Updates pylons config with new settings from database
289 299
290 :param config: 300 :param config:
291 """ 301 """
292 hgsettings = RhodeCodeSettings.get_app_settings() 302 hgsettings = RhodeCodeSetting.get_app_settings()
293 303
294 for k, v in hgsettings.items(): 304 for k, v in hgsettings.items():
295 config[k] = v 305 config[k] = v
296 306
297 307
298 def invalidate_cache(cache_key, *args): 308 def invalidate_cache(cache_key, *args):
299 """Puts cache invalidation task into db for 309 """
310 Puts cache invalidation task into db for
300 further global cache invalidation 311 further global cache invalidation
301 """ 312 """
302 313
303 from rhodecode.model.scm import ScmModel 314 from rhodecode.model.scm import ScmModel
304 315
311 """ 322 """
312 An dummy empty changeset. It's possible to pass hash when creating 323 An dummy empty changeset. It's possible to pass hash when creating
313 an EmptyChangeset 324 an EmptyChangeset
314 """ 325 """
315 326
316 def __init__(self, cs='0' * 40, repo=None, requested_revision=None, alias=None): 327 def __init__(self, cs='0' * 40, repo=None, requested_revision=None,
328 alias=None):
317 self._empty_cs = cs 329 self._empty_cs = cs
318 self.revision = -1 330 self.revision = -1
319 self.message = '' 331 self.message = ''
320 self.author = '' 332 self.author = ''
321 self.date = '' 333 self.date = ''
323 self.requested_revision = requested_revision 335 self.requested_revision = requested_revision
324 self.alias = alias 336 self.alias = alias
325 337
326 @LazyProperty 338 @LazyProperty
327 def raw_id(self): 339 def raw_id(self):
328 """Returns raw string identifying this changeset, useful for web 340 """
341 Returns raw string identifying this changeset, useful for web
329 representation. 342 representation.
330 """ 343 """
331 344
332 return self._empty_cs 345 return self._empty_cs
333 346
348 def get_file_size(self, path): 361 def get_file_size(self, path):
349 return 0 362 return 0
350 363
351 364
352 def map_groups(groups): 365 def map_groups(groups):
353 """Checks for groups existence, and creates groups structures. 366 """
367 Checks for groups existence, and creates groups structures.
354 It returns last group in structure 368 It returns last group in structure
355 369
356 :param groups: list of groups structure 370 :param groups: list of groups structure
357 """ 371 """
358 sa = meta.Session() 372 sa = meta.Session
359 373
360 parent = None 374 parent = None
361 group = None 375 group = None
362 376
363 # last element is repo in nested groups structure 377 # last element is repo in nested groups structure
364 groups = groups[:-1] 378 groups = groups[:-1]
365 379 rgm = ReposGroupModel(sa)
366 for lvl, group_name in enumerate(groups): 380 for lvl, group_name in enumerate(groups):
367 group_name = '/'.join(groups[:lvl] + [group_name]) 381 group_name = '/'.join(groups[:lvl] + [group_name])
368 group = sa.query(Group).filter(Group.group_name == group_name).scalar() 382 group = RepoGroup.get_by_group_name(group_name)
383 desc = '%s group' % group_name
384
385 # # WTF that doesn't work !?
386 # if group is None:
387 # group = rgm.create(group_name, desc, parent, just_db=True)
388 # sa.commit()
369 389
370 if group is None: 390 if group is None:
371 group = Group(group_name, parent) 391 log.debug('creating group level: %s group_name: %s' % (lvl, group_name))
392 group = RepoGroup(group_name, parent)
393 group.group_description = desc
372 sa.add(group) 394 sa.add(group)
395 rgm._create_default_perms(group)
373 sa.commit() 396 sa.commit()
374 parent = group 397 parent = group
375 return group 398 return group
376 399
377 400
378 def repo2db_mapper(initial_repo_list, remove_obsolete=False): 401 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
379 """maps all repos given in initial_repo_list, non existing repositories 402 """
403 maps all repos given in initial_repo_list, non existing repositories
380 are created, if remove_obsolete is True it also check for db entries 404 are created, if remove_obsolete is True it also check for db entries
381 that are not in initial_repo_list and removes them. 405 that are not in initial_repo_list and removes them.
382 406
383 :param initial_repo_list: list of repositories found by scanning methods 407 :param initial_repo_list: list of repositories found by scanning methods
384 :param remove_obsolete: check for obsolete entries in database 408 :param remove_obsolete: check for obsolete entries in database
385 """ 409 """
386 410 from rhodecode.model.repo import RepoModel
387 sa = meta.Session() 411 sa = meta.Session
388 rm = RepoModel() 412 rm = RepoModel()
389 user = sa.query(User).filter(User.admin == True).first() 413 user = sa.query(User).filter(User.admin == True).first()
414 if user is None:
415 raise Exception('Missing administrative account !')
390 added = [] 416 added = []
391 # fixup groups paths to new format on the fly 417
392 # TODO: remove this in future
393 for g in Group.query().all():
394 g.group_name = g.get_new_name(g.name)
395 sa.add(g)
396 for name, repo in initial_repo_list.items(): 418 for name, repo in initial_repo_list.items():
397 group = map_groups(name.split(Repository.url_sep())) 419 group = map_groups(name.split(Repository.url_sep()))
398 if not rm.get_by_repo_name(name, cache=False): 420 if not rm.get_by_repo_name(name, cache=False):
399 log.info('repository %s not found creating default', name) 421 log.info('repository %s not found creating default' % name)
400 added.append(name) 422 added.append(name)
401 form_data = { 423 form_data = {
402 'repo_name': name, 424 'repo_name': name,
403 'repo_name_full': name, 425 'repo_name_full': name,
404 'repo_type': repo.alias, 426 'repo_type': repo.alias,
405 'description': repo.description \ 427 'description': repo.description \
406 if repo.description != 'unknown' else \ 428 if repo.description != 'unknown' else '%s repository' % name,
407 '%s repository' % name, 429 'private': False,
408 'private': False, 430 'group_id': getattr(group, 'group_id', None)
409 'group_id': getattr(group, 'group_id', None) 431 }
410 }
411 rm.create(form_data, user, just_db=True) 432 rm.create(form_data, user, just_db=True)
412 433 sa.commit()
413 removed = [] 434 removed = []
414 if remove_obsolete: 435 if remove_obsolete:
415 #remove from database those repositories that are not in the filesystem 436 #remove from database those repositories that are not in the filesystem
416 for repo in sa.query(Repository).all(): 437 for repo in sa.query(Repository).all():
417 if repo.repo_name not in initial_repo_list.keys(): 438 if repo.repo_name not in initial_repo_list.keys():
419 sa.delete(repo) 440 sa.delete(repo)
420 sa.commit() 441 sa.commit()
421 442
422 return added, removed 443 return added, removed
423 444
424 #set cache regions for beaker so celery can utilise it 445
446 # set cache regions for beaker so celery can utilise it
425 def add_cache(settings): 447 def add_cache(settings):
426 cache_settings = {'regions': None} 448 cache_settings = {'regions': None}
427 for key in settings.keys(): 449 for key in settings.keys():
428 for prefix in ['beaker.cache.', 'cache.']: 450 for prefix in ['beaker.cache.', 'cache.']:
429 if key.startswith(prefix): 451 if key.startswith(prefix):
453 # TEST FUNCTIONS AND CREATORS 475 # TEST FUNCTIONS AND CREATORS
454 #============================================================================== 476 #==============================================================================
455 def create_test_index(repo_location, config, full_index): 477 def create_test_index(repo_location, config, full_index):
456 """ 478 """
457 Makes default test index 479 Makes default test index
458 480
459 :param config: test config 481 :param config: test config
460 :param full_index: 482 :param full_index:
461 """ 483 """
462 484
463 from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon 485 from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
478 except LockHeld: 500 except LockHeld:
479 pass 501 pass
480 502
481 503
482 def create_test_env(repos_test_path, config): 504 def create_test_env(repos_test_path, config):
483 """Makes a fresh database and 505 """
506 Makes a fresh database and
484 install test repository into tmp dir 507 install test repository into tmp dir
485 """ 508 """
486 from rhodecode.lib.db_manage import DbManage 509 from rhodecode.lib.db_manage import DbManage
487 from rhodecode.tests import HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, \ 510 from rhodecode.tests import HG_REPO, TESTS_TMP_PATH
488 HG_FORK, GIT_FORK, TESTS_TMP_PATH
489 import tarfile
490 import shutil
491 from os.path import abspath
492 511
493 # PART ONE create db 512 # PART ONE create db
494 dbconf = config['sqlalchemy.db1.url'] 513 dbconf = config['sqlalchemy.db1.url']
495 log.debug('making test db %s', dbconf) 514 log.debug('making test db %s' % dbconf)
496 515
497 # create test dir if it doesn't exist 516 # create test dir if it doesn't exist
498 if not os.path.isdir(repos_test_path): 517 if not os.path.isdir(repos_test_path):
499 log.debug('Creating testdir %s' % repos_test_path) 518 log.debug('Creating testdir %s' % repos_test_path)
500 os.makedirs(repos_test_path) 519 os.makedirs(repos_test_path)
505 dbmanage.create_settings(dbmanage.config_prompt(repos_test_path)) 524 dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
506 dbmanage.create_default_user() 525 dbmanage.create_default_user()
507 dbmanage.admin_prompt() 526 dbmanage.admin_prompt()
508 dbmanage.create_permissions() 527 dbmanage.create_permissions()
509 dbmanage.populate_default_permissions() 528 dbmanage.populate_default_permissions()
510 529 Session.commit()
511 # PART TWO make test repo 530 # PART TWO make test repo
512 log.debug('making test vcs repositories') 531 log.debug('making test vcs repositories')
513 532
514 idx_path = config['app_conf']['index_dir'] 533 idx_path = config['app_conf']['index_dir']
515 data_path = config['app_conf']['cache_dir'] 534 data_path = config['app_conf']['cache_dir']
593 from pylons import config as pylonsconfig 612 from pylons import config as pylonsconfig
594 613
595 path_to_ini_file = os.path.realpath(conf) 614 path_to_ini_file = os.path.realpath(conf)
596 conf = paste.deploy.appconfig('config:' + path_to_ini_file) 615 conf = paste.deploy.appconfig('config:' + path_to_ini_file)
597 pylonsconfig.init_app(conf.global_conf, conf.local_conf) 616 pylonsconfig.init_app(conf.global_conf, conf.local_conf)
598