Mercurial > kallithea
changeset 8268:f8f50d3b6512
celery: upgrade to Celery 4
Celery 3 doesn't support Python 3.7 or later. This upgrade is thus essential
for full Python 3 support. But note that
https://docs.celeryproject.org/en/4.4.0/faq.html#does-celery-support-windows
says "No".
The names of config settings changed in Celery 3 to 4, as described on
https://docs.celeryproject.org/en/3.0/whatsnew-4.0.html#lowercase-setting-names .
Celery 4 config settings can now be specified in Kallithea .ini files by prefixing
with `celery.` - for example as `celery.broker_url`.
Remain backwards compatible for the usual settings, and map old names to the
new names.
| author | Mads Kiilerich <mads@kiilerich.com> |
|---|---|
| date | Thu, 20 Feb 2020 02:03:24 +0100 |
| parents | e432a6a7dd7c |
| children | e1d4a0d8520f |
| files | development.ini docs/setup.rst kallithea/lib/celerypylons/__init__.py kallithea/lib/paster_commands/template.ini.mako setup.py |
| diffstat | 5 files changed, 34 insertions(+), 17 deletions(-) [+] |
line wrap: on
line diff
--- a/development.ini Mon Feb 24 18:37:31 2020 +0100 +++ b/development.ini Thu Feb 20 02:03:24 2020 +0100 @@ -250,6 +250,7 @@ ### CELERY CONFIG #### #################################### +## Note: Celery doesn't support Windows. use_celery = false ## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':
--- a/docs/setup.rst Mon Feb 24 18:37:31 2020 +0100 +++ b/docs/setup.rst Thu Feb 20 02:03:24 2020 +0100 @@ -332,11 +332,11 @@ use_celery = true -and add or change the ``celery.*`` and ``broker.*`` configuration variables. +and add or change the ``celery.*`` configuration variables. -Remember that the ini files use the format with '.' and not with '_' like -Celery. So for example setting `BROKER_HOST` in Celery means setting -`broker.host` in the configuration file. +Configuration settings are prefixed with 'celery.', so for example setting +`broker_url` in Celery means setting `celery.broker_url` in the configuration +file. To start the Celery process, run::
--- a/kallithea/lib/celerypylons/__init__.py Mon Feb 24 18:37:31 2020 +0100 +++ b/kallithea/lib/celerypylons/__init__.py Thu Feb 20 02:03:24 2020 +0100 @@ -23,11 +23,24 @@ class CeleryConfig(object): - CELERY_IMPORTS = ['kallithea.lib.celerylib.tasks'] - CELERY_ACCEPT_CONTENT = ['json'] - CELERY_RESULT_SERIALIZER = 'json' - CELERY_TASK_SERIALIZER = 'json' - CELERY_ALWAYS_EAGER = False + imports = ['kallithea.lib.celerylib.tasks'] + task_always_eager = False + +# map from Kallithea .ini Celery 3 config names to Celery 4 config names +celery3_compat = { + 'broker.url': 'broker_url', + 'celery.accept.content': 'accept_content', + 'celery.always.eager': 'task_always_eager', + 'celery.amqp.task.result.expires': 'result_expires', + 'celeryd.concurrency': 'worker_concurrency', + 'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child', + #'celery.imports' ends up unchanged + 'celery.result.backend': 'result_backend', + 'celery.result.serializer': 'result_serializer', + 'celery.task.serializer': 'task_serializer', +} + +list_config_names = """imports accept_content""".split() desupported = set([ @@ -45,18 +58,20 @@ celery_config = CeleryConfig() - PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split() - LIST_PARAMS = """CELERY_IMPORTS CELERY_ACCEPT_CONTENT""".split() - for config_key, config_value in sorted(config.items()): if config_key in desupported and config_value: log.error('Celery configuration setting %r is no longer supported', config_key) - celery_key = config_key.replace('.', '_').upper() - if celery_key.split('_', 1)[0] not in PREFIXES: + celery_key = celery3_compat.get(config_key) + parts = config_key.split('.', 1) + if celery_key: # explicit Celery 3 backwards compatibility + pass + elif parts[0] == 'celery' and len(parts) == 2: # Celery 4 config key + celery_key = parts[1] + else: continue if not isinstance(config_value, str): continue - if celery_key in LIST_PARAMS: + if celery_key in list_config_names: celery_value = config_value.split() elif config_value.isdigit(): celery_value = int(config_value) @@ -72,6 +87,6 @@ """Create celery app from the TurboGears configuration file""" app = celery.Celery() celery_config = make_celery_config(tg.config) - kallithea.CELERY_EAGER = celery_config.CELERY_ALWAYS_EAGER + kallithea.CELERY_EAGER = celery_config.task_always_eager app.config_from_object(celery_config) return app
--- a/kallithea/lib/paster_commands/template.ini.mako Mon Feb 24 18:37:31 2020 +0100 +++ b/kallithea/lib/paster_commands/template.ini.mako Thu Feb 20 02:03:24 2020 +0100 @@ -356,6 +356,7 @@ <%text>### CELERY CONFIG ####</%text> <%text>####################################</%text> +<%text>## Note: Celery doesn't support Windows.</%text> use_celery = false <%text>## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':</%text>
--- a/setup.py Mon Feb 24 18:37:31 2020 +0100 +++ b/setup.py Thu Feb 20 02:03:24 2020 +0100 @@ -54,7 +54,7 @@ "Mako >= 0.9.1, < 1.2", "Pygments >= 2.2.0, < 2.6", "Whoosh >= 2.7.1, < 2.8", - "celery >= 3.1, < 4.0", # TODO: celery 4 doesn't work + "celery >= 4.3, < 4.5", "Babel >= 1.3, < 2.9", "python-dateutil >= 2.1.0, < 2.9", "Markdown >= 2.2.1, < 3.2",