Mercurial > kallithea
changeset 8779:ea1c608efa3a
celery: rename celerypylons to more descriptive celery_app
This module is all about creating a Celery app.
author | Mads Kiilerich <mads@kiilerich.com> |
---|---|
date | Sat, 07 Nov 2020 18:49:57 +0100 |
parents | 341e4bb9e227 |
children | eed44652346d |
files | kallithea/config/app_cfg.py kallithea/lib/celery_app.py kallithea/lib/celerypylons/__init__.py |
diffstat | 3 files changed, 94 insertions(+), 94 deletions(-) [+] |
line wrap: on
line diff
--- a/kallithea/config/app_cfg.py Sat Nov 07 18:42:58 2020 +0100 +++ b/kallithea/config/app_cfg.py Sat Nov 07 18:49:57 2020 +0100 @@ -33,7 +33,7 @@ import kallithea.lib.locales import kallithea.model.base import kallithea.model.meta -from kallithea.lib import celerypylons +from kallithea.lib import celery_app from kallithea.lib.utils import load_extensions, set_app_settings, set_indexer_config, set_vcs_config from kallithea.lib.utils2 import asbool, check_git_version from kallithea.model import db @@ -136,7 +136,7 @@ kallithea.DEFAULT_USER_ID = db.User.get_default_user().user_id if asbool(config.get('use_celery')): - kallithea.CELERY_APP = celerypylons.make_app() + kallithea.CELERY_APP = celery_app.make_celery_app() kallithea.CONFIG = config load_extensions(root_path=config['here'])
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kallithea/lib/celery_app.py Sat Nov 07 18:49:57 2020 +0100 @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +""" +Kallithea wrapper of Celery + +The Celery configuration is in the Kallithea ini file but must be converted to an +entirely different format before Celery can use it. + +We read the configuration from tg.config at module import time. This module can +thus not be imported in global scope but must be imported on demand in function +scope after tg.config has been initialized. + +To make sure that the config really has been initialized, we check one of the +mandatory settings. +""" + +import logging + +import celery +import tg + +import kallithea + + +class CeleryConfig(object): + imports = ['kallithea.model.async_tasks'] + task_always_eager = False + +# map from Kallithea .ini Celery 3 config names to Celery 4 config names +celery3_compat = { + 'broker.url': 'broker_url', + 'celery.accept.content': 'accept_content', + 'celery.always.eager': 'task_always_eager', + 'celery.amqp.task.result.expires': 'result_expires', + 'celeryd.concurrency': 'worker_concurrency', + 'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child', + #'celery.imports' ends up unchanged + 'celery.result.backend': 'result_backend', + 'celery.result.serializer': 'result_serializer', + 'celery.task.serializer': 'task_serializer', +} + +list_config_names = """imports accept_content""".split() + + +desupported = set([ + 'celery.result.dburi', + 'celery.result.serialier', + 'celery.send.task.error.emails', +]) + + +log = logging.getLogger(__name__) + + +def make_celery_config(config): + """Return Celery config object populated from relevant settings in a config dict, such as tg.config""" + + celery_config = CeleryConfig() + + for config_key, config_value in sorted(config.items()): + if config_key in desupported and config_value: + log.error('Celery configuration setting %r is no longer supported', config_key) + celery_key = celery3_compat.get(config_key) + parts = config_key.split('.', 1) + if celery_key: # explicit Celery 3 backwards compatibility + pass + elif parts[0] == 'celery' and len(parts) == 2: # Celery 4 config key + celery_key = parts[1] + else: + continue + if not isinstance(config_value, str): + continue + if celery_key in list_config_names: + celery_value = config_value.split() + elif config_value.isdigit(): + celery_value = int(config_value) + elif config_value.lower() in ['true', 'false']: + celery_value = config_value.lower() == 'true' + else: + celery_value = config_value + setattr(celery_config, celery_key, celery_value) + return celery_config + + +def make_app(): + """Create celery app from the TurboGears configuration file""" + app = celery.Celery() + celery_config = make_celery_config(tg.config) + kallithea.CELERY_EAGER = celery_config.task_always_eager + app.config_from_object(celery_config) + return app
--- a/kallithea/lib/celerypylons/__init__.py Sat Nov 07 18:42:58 2020 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,92 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Kallithea wrapper of Celery - -The Celery configuration is in the Kallithea ini file but must be converted to an -entirely different format before Celery can use it. - -We read the configuration from tg.config at module import time. This module can -thus not be imported in global scope but must be imported on demand in function -scope after tg.config has been initialized. - -To make sure that the config really has been initialized, we check one of the -mandatory settings. -""" - -import logging - -import celery -import tg - -import kallithea - - -class CeleryConfig(object): - imports = ['kallithea.model.async_tasks'] - task_always_eager = False - -# map from Kallithea .ini Celery 3 config names to Celery 4 config names -celery3_compat = { - 'broker.url': 'broker_url', - 'celery.accept.content': 'accept_content', - 'celery.always.eager': 'task_always_eager', - 'celery.amqp.task.result.expires': 'result_expires', - 'celeryd.concurrency': 'worker_concurrency', - 'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child', - #'celery.imports' ends up unchanged - 'celery.result.backend': 'result_backend', - 'celery.result.serializer': 'result_serializer', - 'celery.task.serializer': 'task_serializer', -} - -list_config_names = """imports accept_content""".split() - - -desupported = set([ - 'celery.result.dburi', - 'celery.result.serialier', - 'celery.send.task.error.emails', -]) - - -log = logging.getLogger(__name__) - - -def make_celery_config(config): - """Return Celery config object populated from relevant settings in a config dict, such as tg.config""" - - celery_config = CeleryConfig() - - for config_key, config_value in sorted(config.items()): - if config_key in desupported and config_value: - log.error('Celery configuration setting %r is no longer supported', config_key) - celery_key = celery3_compat.get(config_key) - parts = config_key.split('.', 1) - if celery_key: # explicit Celery 3 backwards compatibility - pass - elif parts[0] == 'celery' and len(parts) == 2: # Celery 4 config key - celery_key = parts[1] - else: - continue - if not isinstance(config_value, str): - continue - if celery_key in list_config_names: - celery_value = config_value.split() - elif config_value.isdigit(): - celery_value = int(config_value) - elif config_value.lower() in ['true', 'false']: - celery_value = config_value.lower() == 'true' - else: - celery_value = config_value - setattr(celery_config, celery_key, celery_value) - return celery_config - - -def make_app(): - """Create celery app from the TurboGears configuration file""" - app = celery.Celery() - celery_config = make_celery_config(tg.config) - kallithea.CELERY_EAGER = celery_config.task_always_eager - app.config_from_object(celery_config) - return app