changeset 6803:3fb0ce6de10d

celery: replace loader.PylonsSettingsProxy with a simple configuration object The proxy was quite complex and did all the conversion at runtime, making it hard to figure out exactly what it was doing. Instead, just create a simple configuration object as described on http://docs.celeryproject.org/en/3.1/userguide/application.html and convert all relevant config settings up front. We don't convert all config settings, but only the ones Celery seems to be interested in. The value conversion heuristics are the same as before and still seem quite fragile.
author Mads Kiilerich <mads@kiilerich.com>
date Sun, 13 Aug 2017 01:59:19 +0200
parents b9853a3cc254
children 2f93ca3eab8b
files kallithea/lib/celerypylons/__init__.py kallithea/lib/celerypylons/loader.py
diffstat 2 files changed, 38 insertions(+), 72 deletions(-) [+]
line wrap: on
line diff
--- a/kallithea/lib/celerypylons/__init__.py	Sun Jun 11 16:13:09 2017 +0200
+++ b/kallithea/lib/celerypylons/__init__.py	Sun Aug 13 01:59:19 2017 +0200
@@ -3,31 +3,57 @@
 """
 Kallithea wrapper of Celery
 
-The Celery configuration is in the ini file. To read the settings and translate
-to a Celery format we use PylonsSettingsProxy.
+The Celery configuration is in the Kallithea ini file but must be converted to an
+entirely different format before Celery can use it.
 
-We read the configuration from tg.config, thus it must be initialized before
-loading this module. To make sure that really is the case and give an early
-warning, we check one of the mandatory settings.
+We read the configuration from tg.config at module import time. This module can
+thus not be imported in global scope but must be imported on demand in function
+scope after tg.config has been initialized.
 
-This module must thus not be imported in global scope but must be imported on
-demand in function scope after tg.config has been initialized.
+To make sure that the config really has been initialized, we check one of the
+mandatory settings.
 """
 
 import os
 import warnings
 
+import tg
 import celery
 
-from kallithea.lib.celerypylons.loader import PylonsSettingsProxy
+
+def celery_config(config):
+    """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
+
+    # Verify .ini file configuration has been loaded
+    assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'
+
+    class CeleryConfig(object): pass
+    celery_config = CeleryConfig()
+
+    PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()
+    LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES CELERY_ACCEPT_CONTENT""".split()
 
-# Verify Pylons configuration has been loaded
-from tg import config
-assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'
+    for config_key, config_value in sorted(config.items()):
+        celery_key = config_key.replace('.', '_').upper()
+        if celery_key.split('_', 1)[0] not in PREFIXES:
+            continue
+        if not isinstance(config_value, basestring):
+            continue
+        if celery_key in LIST_PARAMS:
+            celery_value = config_value.split()
+        elif config_value.isdigit():
+            celery_value = int(config_value)
+        elif config_value.lower() in ['true', 'false']:
+            celery_value = config_value.lower() == 'true'
+        else:
+            celery_value = config_value
+        setattr(celery_config, celery_key, celery_value)
+    return celery_config
+
 
 # Create celery app from the TurboGears configuration file
 app = celery.Celery()
-app.config_from_object(PylonsSettingsProxy())
+app.config_from_object(celery_config(tg.config))
 
 import celery.result as result
 from celery.task import task
--- a/kallithea/lib/celerypylons/loader.py	Sun Jun 11 16:13:09 2017 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,60 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from tg import config
-
-# TODO: drop this mangling and just use a separate celery config section
-to_pylons = lambda x: x.replace('_', '.').lower()
-to_celery = lambda x: x.replace('.', '_').upper()
-
-LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES CELERY_ACCEPT_CONTENT""".split()
-
-
-class PylonsSettingsProxy(object):
-    """Pylons Settings Proxy
-
-    Make settings from pylons.config appear the way Celery expects them.
-    """
-
-    def __getattr__(self, key):
-        try:
-            return self[key]
-        except KeyError:
-            raise AttributeError(key)
-
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-
-    def __getitem__(self, key):
-        pylons_key = to_pylons(key)
-        value = config[pylons_key]
-        if key in LIST_PARAMS:
-            return value.split()
-        return self.type_converter(value)
-
-    def __contains__(self, key):
-        pylons_key = to_pylons(key)
-        try:
-            config[pylons_key]
-        except KeyError:
-            return False
-        return True
-
-    def __setattr__(self, key, value):
-        pylons_key = to_pylons(key)
-        config[pylons_key] = value
-
-    def __setitem__(self, key, value):
-        self.__setattr__(key, value)
-
-    def type_converter(self, value):
-        #cast to int
-        if value.isdigit():
-            return int(value)
-
-        #cast to bool
-        if value.lower() in ['true', 'false']:
-            return value.lower() == 'true'
-        return value