Mercurial > kallithea
changeset 6172:7e7db11d4e4d
celerypylons: wrap celery import so we always get the right environment variables set and check configuration
Get rid of magic ... or at least document and encapsulate it.
Before, celerypylons would set the environment variables that made it possible
to import Celery after Pylons had been configured.
Now, the module will import Celery ... and verify that Pylons has been
configured.
(A next stop could be to move things around so this got tied closely to
initializing Pylons. Or something.)
| author | Mads Kiilerich <madski@unity3d.com> |
|---|---|
| date | Tue, 06 Sep 2016 00:51:18 +0200 |
| parents | 54545cc34c36 |
| children | cf73bd884a53 |
| files | kallithea/config/environment.py kallithea/controllers/admin/repos.py kallithea/lib/celerylib/__init__.py kallithea/lib/celerypylons/__init__.py kallithea/lib/celerypylons/commands.py kallithea/lib/paster_commands/common.py |
| diffstat | 6 files changed, 35 insertions(+), 27 deletions(-) [+] |
line wrap: on
line diff
--- a/kallithea/config/environment.py Tue Sep 06 00:51:18 2016 +0200 +++ b/kallithea/config/environment.py Tue Sep 06 00:51:18 2016 +0200 @@ -25,9 +25,6 @@ import beaker import formencode -# don't remove this import it does magic for celery -from kallithea.lib import celerypylons - import kallithea.lib.app_globals as app_globals from kallithea.config.routing import make_map
--- a/kallithea/controllers/admin/repos.py Tue Sep 06 00:51:18 2016 +0200 +++ b/kallithea/controllers/admin/repos.py Tue Sep 06 00:51:18 2016 +0200 @@ -189,9 +189,9 @@ if task_id and task_id not in ['None']: from kallithea import CELERY_ON - from celery.result import AsyncResult + from kallithea.lib import celerypylons if CELERY_ON: - task = AsyncResult(task_id) + task = celerypylons.result.AsyncResult(task_id) if task.failed(): raise HTTPInternalServerError(task.traceback)
--- a/kallithea/lib/celerylib/__init__.py Tue Sep 06 00:51:18 2016 +0200 +++ b/kallithea/lib/celerylib/__init__.py Tue Sep 06 00:51:18 2016 +0200 @@ -72,8 +72,8 @@ finally: log.info('executed %s task', f_org.__name__) f_async.__name__ = f_org.__name__ - import celery.task - runner = celery.task.task(ignore_result=True)(f_async) + from kallithea.lib import celerypylons + runner = celerypylons.task(ignore_result=True)(f_async) def f_wrapped(*args, **kwargs): t = runner.apply_async(args=args, kwargs=kwargs) log.info('executing task %s in async mode - id %s', f_org, t.task_id)
--- a/kallithea/lib/celerypylons/__init__.py Tue Sep 06 00:51:18 2016 +0200 +++ b/kallithea/lib/celerypylons/__init__.py Tue Sep 06 00:51:18 2016 +0200 @@ -1,19 +1,36 @@ # -*- coding: utf-8 -*- """ -Automatically sets the environment variable `CELERY_LOADER` to -`celerypylons.loader:PylonsLoader`. This ensures the loader is -specified when accessing the rest of this package, and allows celery -to be installed in a webapp just by importing celerypylons:: +Kallithea wrapper of Celery + +The Celery configuration is in the normal Pylons ini file. We thus have to set +the `CELERY_LOADER` environment variable to point at a custom "loader" that can +read it. That environment variable must be set *before* importing celery. To +ensure that, we wrap celery in this module. - import celerypylons +Also, the loader depends on Pylons being configured to it can read the Celery +configuration out of it. To make sure that really is the case and give an early +warning, we check one of the mandatory settings. +This module must thus not be imported in global scope but must be imported on +demand in function scope. """ import os import warnings +# Verify Pylons configuration has been loaded +from pylons import config +assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded' + +# Prepare environment to point at Kallithea Pylons loader CELERYPYLONS_LOADER = 'kallithea.lib.celerypylons.loader.PylonsLoader' if os.environ.get('CELERY_LOADER', CELERYPYLONS_LOADER) != CELERYPYLONS_LOADER: warnings.warn("'CELERY_LOADER' environment variable will be overridden by celery-pylons.") os.environ['CELERY_LOADER'] = CELERYPYLONS_LOADER + +# Import (and expose) celery, thus immediately triggering use of the custom Pylons loader +import celery.app as app +import celery.result as result +from celery.task import task +from celery.bin import camqadm, celerybeat, celeryd, celeryev
--- a/kallithea/lib/celerypylons/commands.py Tue Sep 06 00:51:18 2016 +0200 +++ b/kallithea/lib/celerypylons/commands.py Tue Sep 06 00:51:18 2016 +0200 @@ -3,8 +3,7 @@ import kallithea from kallithea.lib.paster_commands.common import BasePasterCommand from kallithea.lib.utils import Command, load_rcextensions -from celery.app import app_or_default -from celery.bin import camqadm, celerybeat, celeryd, celeryev + from kallithea.lib.utils2 import str2bool @@ -26,12 +25,13 @@ allow options/arguments to be passed through to the underlying celery command. """ - - cmd = self.celery_command(app_or_default()) + from kallithea.lib import celerypylons + cmd = self.celery_command(celerypylons.app.app_or_default()) for x in cmd.get_options(): self.parser.add_option(x) def command(self): + from kallithea.lib import celerypylons from pylons import config try: CELERY_ON = str2bool(config['app_conf'].get('use_celery')) @@ -43,7 +43,7 @@ 'file before running celeryd') kallithea.CELERY_ON = CELERY_ON load_rcextensions(config['here']) - cmd = self.celery_command(app_or_default()) + cmd = self.celery_command(celerypylons.app.app_or_default()) return cmd.run(**vars(self.options)) @@ -58,7 +58,7 @@ description = "".join(__doc__.splitlines()[2:]) parser = Command.standard_parser(quiet=True) - celery_command = celeryd.WorkerCommand + celery_command = celerypylons.celeryd.WorkerCommand class CeleryBeatCommand(CeleryCommand): @@ -72,7 +72,7 @@ description = "".join(__doc__.splitlines()[2:]) parser = Command.standard_parser(quiet=True) - celery_command = celerybeat.BeatCommand + celery_command = celerypylons.celerybeat.BeatCommand class CAMQPAdminCommand(CeleryCommand): @@ -85,7 +85,7 @@ description = "".join(__doc__.splitlines()[2:]) parser = Command.standard_parser(quiet=True) - celery_command = camqadm.AMQPAdminCommand + celery_command = celerypylons.camqadm.AMQPAdminCommand class CeleryEventCommand(CeleryCommand): @@ -98,4 +98,4 @@ description = "".join(__doc__.splitlines()[2:]) parser = Command.standard_parser(quiet=True) - celery_command = celeryev.EvCommand + celery_command = celerypylons.celeryev.EvCommand
--- a/kallithea/lib/paster_commands/common.py Tue Sep 06 00:51:18 2016 +0200 +++ b/kallithea/lib/paster_commands/common.py Tue Sep 06 00:51:18 2016 +0200 @@ -50,12 +50,6 @@ class BasePasterCommand(Command): """ Abstract Base Class for paster commands. - - The celery commands are somewhat aggressive about loading - celery.conf, and since our module sets the `CELERY_LOADER` - environment variable to our loader, we have to bootstrap a bit and - make sure we've had a chance to load the pylons config off of the - command line, otherwise everything fails. """ min_args = 1 min_args_error = "Please provide a paster config file as an argument."