Mercurial > kallithea
changeset 8831:1f92cded1bd2
celery: move Whoosh indexing task to whoosh library where it belongs
Avoid bundling everything from many different layers in one big task library.
This is more feasible now when we don't need kallithea.CELERY_APP set at import
time.
| author | Mads Kiilerich <mads@kiilerich.com> |
|---|---|
| date | Tue, 29 Dec 2020 22:07:41 +0100 |
| parents | 853717af31d3 |
| children | 1d3b67443aac |
| files | kallithea/controllers/admin/settings.py kallithea/lib/celery_app.py kallithea/lib/indexers/daemon.py kallithea/model/async_tasks.py scripts/deps.py |
| diffstat | 5 files changed, 18 insertions(+), 14 deletions(-) [+] |
line wrap: on
line diff
--- a/kallithea/controllers/admin/settings.py Fri Jan 01 18:04:16 2021 +0100 +++ b/kallithea/controllers/admin/settings.py Tue Dec 29 22:07:41 2020 +0100 @@ -36,6 +36,7 @@ from webob.exc import HTTPFound import kallithea +import kallithea.lib.indexers.daemon from kallithea.controllers import base from kallithea.lib import webutils from kallithea.lib.auth import HasPermissionAnyDecorator, LoginRequired @@ -378,7 +379,7 @@ if request.POST: repo_location = self._get_hg_ui_settings()['paths_root_path'] full_index = request.POST.get('full_index', False) - async_tasks.whoosh_index(repo_location, full_index) + kallithea.lib.indexers.daemon.whoosh_index(repo_location, full_index) webutils.flash(_('Whoosh reindex task scheduled'), category='success') raise HTTPFound(location=url('admin_settings_search'))
--- a/kallithea/lib/celery_app.py Fri Jan 01 18:04:16 2021 +0100 +++ b/kallithea/lib/celery_app.py Tue Dec 29 22:07:41 2020 +0100 @@ -18,7 +18,10 @@ class CeleryConfig(object): - imports = ['kallithea.model.async_tasks'] + imports = [ + 'kallithea.lib.indexers.daemon', + 'kallithea.model.async_tasks', + ] task_always_eager = False list_config_names = {'imports', 'accept_content'}
--- a/kallithea/lib/indexers/daemon.py Fri Jan 01 18:04:16 2021 +0100 +++ b/kallithea/lib/indexers/daemon.py Tue Dec 29 22:07:41 2020 +0100 @@ -32,9 +32,11 @@ from shutil import rmtree from time import mktime +from tg import config from whoosh.index import create_in, exists_in, open_dir from whoosh.qparser import QueryParser +from kallithea.lib import celerylib from kallithea.lib.conf import INDEX_EXTENSIONS, INDEX_FILENAMES from kallithea.lib.indexers import CHGSET_IDX_NAME, CHGSETS_SCHEMA, IDX_NAME, SCHEMA from kallithea.lib.utils2 import safe_str @@ -444,3 +446,12 @@ self.build_indexes() else: self.update_indexes() + + +@celerylib.task +@celerylib.locked_task +def whoosh_index(repo_location, full_index): + index_location = config['index_dir'] + WhooshIndexingDaemon(index_location=index_location, + repo_location=repo_location) \ + .run(full_index=full_index)
--- a/kallithea/model/async_tasks.py Fri Jan 01 18:04:16 2021 +0100 +++ b/kallithea/model/async_tasks.py Tue Dec 29 22:07:41 2020 +0100 @@ -41,27 +41,17 @@ import kallithea from kallithea.lib import celerylib, conf, ext_json, hooks -from kallithea.lib.indexers.daemon import WhooshIndexingDaemon from kallithea.lib.utils2 import asbool, ascii_bytes from kallithea.lib.vcs.utils import author_email, author_name from kallithea.model import db, meta, repo, userlog -__all__ = ['whoosh_index', 'get_commits_stats', 'send_email'] +__all__ = ['get_commits_stats', 'send_email'] log = celery.utils.log.get_task_logger(__name__) -@celerylib.task -@celerylib.locked_task -def whoosh_index(repo_location, full_index): - index_location = config['index_dir'] - WhooshIndexingDaemon(index_location=index_location, - repo_location=repo_location) \ - .run(full_index=full_index) - - def _author_username(author): """Return the username of the user identified by the email part of the 'author' string, default to the name or email.
--- a/scripts/deps.py Fri Jan 01 18:04:16 2021 +0100 +++ b/scripts/deps.py Tue Dec 29 22:07:41 2020 +0100 @@ -159,7 +159,6 @@ ('kallithea.lib.utils', 'kallithea.model.db'), ('kallithea.lib.utils', 'kallithea.model.scm'), ('kallithea.model.async_tasks', 'kallithea.lib.hooks'), -('kallithea.model.async_tasks', 'kallithea.lib.indexers'), ('kallithea.model.async_tasks', 'kallithea.model'), ('kallithea.model', 'kallithea.lib.auth'), # auth.HasXXX ('kallithea.model', 'kallithea.lib.auth_modules'), # validators