Mercurial > kallithea
changeset 785:277427ac29a9 beta
complete rewrite of paster commands,
used fancy celery-pylons baseCommand to implement better support for paster commands
fixed make-index, and paster celeryd. Moved config's add_cache to utils
| author | Marcin Kuzminski <marcin@python-works.com> |
|---|---|
| date | Sun, 28 Nov 2010 05:31:05 +0100 |
| parents | 30d3161c6683 |
| children | 2889a4446960 |
| files | rhodecode/lib/celerylib/tasks.py rhodecode/lib/celerypylons/commands.py rhodecode/lib/indexers/__init__.py rhodecode/lib/utils.py |
| diffstat | 4 files changed, 131 insertions(+), 116 deletions(-) [+] |
line wrap: on
line diff
--- a/rhodecode/lib/celerylib/tasks.py Sun Nov 28 05:26:26 2010 +0100 +++ b/rhodecode/lib/celerylib/tasks.py Sun Nov 28 05:31:05 2010 +0100 @@ -2,7 +2,6 @@ import os import traceback -import beaker from time import mktime from operator import itemgetter @@ -12,7 +11,7 @@ from rhodecode.lib.celerylib import run_task, locked_task, str2bool from rhodecode.lib.helpers import person from rhodecode.lib.smtp_mailer import SmtpMailer -from rhodecode.lib.utils import OrderedDict +from rhodecode.lib.utils import OrderedDict, add_cache from rhodecode.model import init_model from rhodecode.model import meta from rhodecode.model.db import RhodeCodeUi @@ -21,29 +20,6 @@ from sqlalchemy import engine_from_config -#set cache regions for beaker so celery can utilise it -def add_cache(settings): - cache_settings = {'regions':None} - for key in settings.keys(): - for prefix in ['beaker.cache.', 'cache.']: - if key.startswith(prefix): - name = key.split(prefix)[1].strip() - cache_settings[name] = settings[key].strip() - if cache_settings['regions']: - for region in cache_settings['regions'].split(','): - region = region.strip() - region_settings = {} - for key, value in cache_settings.items(): - if key.startswith(region): - region_settings[key.split('.')[1]] = value - region_settings['expire'] = int(region_settings.get('expire', - 60)) - region_settings.setdefault('lock_dir', - cache_settings.get('lock_dir')) - if 'type' not in region_settings: - region_settings['type'] = cache_settings.get('type', - 'memory') - beaker.cache.cache_regions[region] = region_settings add_cache(config) try:
--- a/rhodecode/lib/celerypylons/commands.py Sun Nov 28 05:26:26 2010 +0100 +++ b/rhodecode/lib/celerypylons/commands.py Sun Nov 28 05:31:05 2010 +0100 @@ -1,64 +1,11 @@ -import os -from paste.script.command import Command, BadCommand -import paste.deploy -from pylons import config +from rhodecode.lib.utils import BasePasterCommand, Command __all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand', 'CAMQPAdminCommand', 'CeleryEventCommand'] -class CeleryCommand(Command): - """ - Abstract Base Class for celery commands. - - The celery commands are somewhat aggressive about loading - celery.conf, and since our module sets the `CELERY_LOADER` - environment variable to our loader, we have to bootstrap a bit and - make sure we've had a chance to load the pylons config off of the - command line, otherwise everything fails. - """ - min_args = 1 - min_args_error = "Please provide a paster config file as an argument." - takes_config_file = 1 - requires_config_file = True - - def run(self, args): - """ - Overrides Command.run - - Checks for a config file argument and loads it. - """ - if len(args) < self.min_args: - raise BadCommand( - self.min_args_error % {'min_args': self.min_args, - 'actual_args': len(args)}) - # Decrement because we're going to lob off the first argument. - # @@ This is hacky - self.min_args -= 1 - self.bootstrap_config(args[0]) - self.update_parser() - return super(CeleryCommand, self).run(args[1:]) - - def update_parser(self): - """ - Abstract method. Allows for the class's parser to be updated - before the superclass's `run` method is called. Necessary to - allow options/arguments to be passed through to the underlying - celery command. - """ - raise NotImplementedError("Abstract Method.") - - def bootstrap_config(self, conf): - """ - Loads the pylons configuration. - """ - path_to_ini_file = os.path.realpath(conf) - conf = paste.deploy.appconfig('config:' + path_to_ini_file) - config.init_app(conf.global_conf, conf.local_conf) - - -class CeleryDaemonCommand(CeleryCommand): +class CeleryDaemonCommand(BasePasterCommand): """Start the celery worker Starts the celery worker that uses a paste.deploy configuration @@ -80,7 +27,7 @@ return celeryd.WorkerCommand().run(**vars(self.options)) -class CeleryBeatCommand(CeleryCommand): +class CeleryBeatCommand(BasePasterCommand): """Start the celery beat server Starts the celery beat server using a paste.deploy configuration @@ -101,7 +48,7 @@ from celery.bin import celerybeat return celerybeat.BeatCommand(**vars(self.options)) -class CAMQPAdminCommand(CeleryCommand): +class CAMQPAdminCommand(BasePasterCommand): """CAMQP Admin CAMQP celery admin tool. @@ -122,7 +69,7 @@ return camqadm.camqadm(*self.args, **vars(self.options)) -class CeleryEventCommand(CeleryCommand): +class CeleryEventCommand(BasePasterCommand): """Celery event commandd. Capture celery events.
--- a/rhodecode/lib/indexers/__init__.py Sun Nov 28 05:26:26 2010 +0100 +++ b/rhodecode/lib/indexers/__init__.py Sun Nov 28 05:31:05 2010 +0100 @@ -1,23 +1,28 @@ import os import sys +import traceback from os.path import dirname as dn, join as jn #to get the rhodecode import sys.path.append(dn(dn(dn(os.path.realpath(__file__))))) +from rhodecode.model import init_model +from rhodecode.model.scm import ScmModel from rhodecode.config.environment import load_environment -from rhodecode.model.scm import ScmModel +from rhodecode.lib.utils import BasePasterCommand, Command, add_cache + from shutil import rmtree from webhelpers.html.builder import escape from vcs.utils.lazy import LazyProperty +from sqlalchemy import engine_from_config + from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter from whoosh.fields import TEXT, ID, STORED, Schema, FieldType from whoosh.index import create_in, open_dir from whoosh.formats import Characters from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter -import traceback #EXTENSIONS WE WANT TO INDEX CONTENT OFF INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', @@ -45,10 +50,8 @@ FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') FRAGMENTER = SimpleFragmenter(200) -from paste.script import command -import ConfigParser -class MakeIndex(command.Command): +class MakeIndex(BasePasterCommand): max_args = 1 min_args = 1 @@ -57,26 +60,16 @@ summary = "Creates index for full text search given configuration file" group_name = "RhodeCode" takes_config_file = -1 - parser = command.Command.standard_parser(verbose=True) - parser.add_option('--repo-location', - action='store', - dest='repo_location', - help="Specifies repositories location to index REQUIRED", - ) - parser.add_option('-f', - action='store_true', - dest='full_index', - help="Specifies that index should be made full i.e" - " destroy old and build from scratch", - default=False) + parser = Command.standard_parser(verbose=True) + def command(self): - config_name = self.args[0] - p = config_name.split('/') - root = '.' if len(p) == 1 else '/'.join(p[:-1]) - config = ConfigParser.ConfigParser({'here':root}) - config.read(config_name) - index_location = dict(config.items('app:main'))['index_dir'] + from pylons import config + add_cache(config) + engine = engine_from_config(config, 'sqlalchemy.db1.') + init_model(engine) + + index_location = config['index_dir'] repo_location = self.options.repo_location #====================================================================== @@ -93,6 +86,18 @@ except LockHeld: sys.exit(1) + def update_parser(self): + self.parser.add_option('--repo-location', + action='store', + dest='repo_location', + help="Specifies repositories location to index REQUIRED", + ) + self.parser.add_option('-f', + action='store_true', + dest='full_index', + help="Specifies that index should be made full i.e" + " destroy old and build from scratch", + default=False) class ResultWrapper(object): def __init__(self, search_type, searcher, matcher, highlight_items):
--- a/rhodecode/lib/utils.py Sun Nov 28 05:26:26 2010 +0100 +++ b/rhodecode/lib/utils.py Sun Nov 28 05:31:05 2010 +0100 @@ -36,7 +36,10 @@ from mercurial import ui, config, hg from mercurial.error import RepoError -from paste.script import command +import paste +import beaker +from paste.script.command import Command, BadCommand + from vcs.backends.base import BaseChangeset from vcs.utils.lazy import LazyProperty @@ -416,6 +419,31 @@ return not self == other +#set cache regions for beaker so celery can utilise it +def add_cache(settings): + cache_settings = {'regions':None} + for key in settings.keys(): + for prefix in ['beaker.cache.', 'cache.']: + if key.startswith(prefix): + name = key.split(prefix)[1].strip() + cache_settings[name] = settings[key].strip() + if cache_settings['regions']: + for region in cache_settings['regions'].split(','): + region = region.strip() + region_settings = {} + for key, value in cache_settings.items(): + if key.startswith(region): + region_settings[key.split('.')[1]] = value + region_settings['expire'] = int(region_settings.get('expire', + 60)) + region_settings.setdefault('lock_dir', + cache_settings.get('lock_dir')) + if 'type' not in region_settings: + region_settings['type'] = cache_settings.get('type', + 'memory') + beaker.cache.cache_regions[region] = region_settings + + #=============================================================================== # TEST FUNCTIONS AND CREATORS #=============================================================================== @@ -498,7 +526,66 @@ tar.extractall(jn(TESTS_TMP_PATH, HG_REPO)) tar.close() -class UpgradeDb(command.Command): + +#============================================================================== +# PASTER COMMANDS +#============================================================================== + +class BasePasterCommand(Command): + """ + Abstract Base Class for paster commands. + + The celery commands are somewhat aggressive about loading + celery.conf, and since our module sets the `CELERY_LOADER` + environment variable to our loader, we have to bootstrap a bit and + make sure we've had a chance to load the pylons config off of the + command line, otherwise everything fails. + """ + min_args = 1 + min_args_error = "Please provide a paster config file as an argument." + takes_config_file = 1 + requires_config_file = True + + def run(self, args): + """ + Overrides Command.run + + Checks for a config file argument and loads it. + """ + if len(args) < self.min_args: + raise BadCommand( + self.min_args_error % {'min_args': self.min_args, + 'actual_args': len(args)}) + + # Decrement because we're going to lob off the first argument. + # @@ This is hacky + self.min_args -= 1 + self.bootstrap_config(args[0]) + self.update_parser() + return super(BasePasterCommand, self).run(args[1:]) + + def update_parser(self): + """ + Abstract method. Allows for the class's parser to be updated + before the superclass's `run` method is called. Necessary to + allow options/arguments to be passed through to the underlying + celery command. + """ + raise NotImplementedError("Abstract Method.") + + def bootstrap_config(self, conf): + """ + Loads the pylons configuration. + """ + from pylons import config as pylonsconfig + + path_to_ini_file = os.path.realpath(conf) + conf = paste.deploy.appconfig('config:' + path_to_ini_file) + pylonsconfig.init_app(conf.global_conf, conf.local_conf) + + + +class UpgradeDb(BasePasterCommand): """Command used for paster to upgrade our database to newer version """ @@ -509,16 +596,16 @@ summary = "Upgrades current db to newer version given configuration file" group_name = "RhodeCode" - parser = command.Command.standard_parser(verbose=True) + parser = Command.standard_parser(verbose=True) - parser.add_option('--sql', + def command(self): + from pylons import config + raise NotImplementedError('Not implemented yet') + + + def update_parser(self): + self.parser.add_option('--sql', action='store_true', dest='just_sql', help="Prints upgrade sql for further investigation", default=False) - def command(self): - config_name = self.args[0] - p = config_name.split('/') - root = '.' if len(p) == 1 else '/'.join(p[:-1]) - config = ConfigParser.ConfigParser({'here':root}) - config.read(config_name)