Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/celery_backend/config.py
| Show All 12 Lines | |||||
| from celery.utils.log import ColorFormatter | from celery.utils.log import ColorFormatter | ||||
| from celery.worker.control import Panel | from celery.worker.control import Panel | ||||
| from kombu import Exchange, Queue | from kombu import Exchange, Queue | ||||
| from kombu.five import monotonic as _monotonic | from kombu.five import monotonic as _monotonic | ||||
| import requests | import requests | ||||
| from typing import Any, Dict | |||||
| from swh.scheduler import CONFIG as SWH_CONFIG | from swh.scheduler import CONFIG as SWH_CONFIG | ||||
| from swh.core.config import load_named_config, merge_configs | from swh.core.config import load_named_config, merge_configs | ||||
| from swh.core.logger import JournalHandler | from swh.core.logger import JournalHandler | ||||
| DEFAULT_CONFIG_NAME = 'worker' | DEFAULT_CONFIG_NAME = 'worker' | ||||
| CONFIG_NAME_ENVVAR = 'SWH_WORKER_INSTANCE' | CONFIG_NAME_ENVVAR = 'SWH_WORKER_INSTANCE' | ||||
| CONFIG_NAME_TEMPLATE = 'worker/%s' | CONFIG_NAME_TEMPLATE = 'worker/%s' | ||||
| ▲ Show 20 Lines • Show All 143 Lines • ▼ Show 20 Lines | def register_task_class(app, name, cls): | ||||
| task_instance = cls() | task_instance = cls() | ||||
| task_instance.name = name | task_instance.name = name | ||||
| app.register_task(task_instance) | app.register_task(task_instance) | ||||
| INSTANCE_NAME = os.environ.get(CONFIG_NAME_ENVVAR) | INSTANCE_NAME = os.environ.get(CONFIG_NAME_ENVVAR) | ||||
| CONFIG_NAME = os.environ.get('SWH_CONFIG_FILENAME') | CONFIG_NAME = os.environ.get('SWH_CONFIG_FILENAME') | ||||
| CONFIG = {} | CONFIG = {} # type: Dict[str, Any] | ||||
| if CONFIG_NAME: | if CONFIG_NAME: | ||||
| # load the celery config from the main config file given as | # load the celery config from the main config file given as | ||||
| # SWH_CONFIG_FILENAME environment variable. | # SWH_CONFIG_FILENAME environment variable. | ||||
| # This is expected to have a [celery] section in which we have the | # This is expected to have a [celery] section in which we have the | ||||
| # celery specific configuration. | # celery specific configuration. | ||||
| SWH_CONFIG.clear() | SWH_CONFIG.clear() | ||||
| SWH_CONFIG.update(load_named_config(CONFIG_NAME)) | SWH_CONFIG.update(load_named_config(CONFIG_NAME)) | ||||
| CONFIG = SWH_CONFIG.get('celery') | CONFIG = SWH_CONFIG.get('celery', default={}) | ||||
ardumont: jsyk, within docker-env, this does not work (after rebuild of the swh containers).
```
swh… | |||||
ardumontUnsubmitted Not Done Inline Actions
There we go D2059 ;) ardumont: > I'll open a diff.
There we go D2059 ;) | |||||
| if not CONFIG: | if not CONFIG: | ||||
| # otherwise, back to compat config loading mechanism | # otherwise, back to compat config loading mechanism | ||||
| if INSTANCE_NAME: | if INSTANCE_NAME: | ||||
| CONFIG_NAME = CONFIG_NAME_TEMPLATE % INSTANCE_NAME | CONFIG_NAME = CONFIG_NAME_TEMPLATE % INSTANCE_NAME | ||||
| else: | else: | ||||
| CONFIG_NAME = DEFAULT_CONFIG_NAME | CONFIG_NAME = DEFAULT_CONFIG_NAME | ||||
| ▲ Show 20 Lines • Show All 85 Lines • Show Last 20 Lines | |||||
jsyk, within docker-env, this does not work (after rebuild of the swh containers).
swh-lister_1 | Traceback (most recent call last): swh-lister_1 | File "/srv/softwareheritage/venv/bin/swh", line 10, in <module> swh-lister_1 | sys.exit(main()) swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/cli/__init__.py", line 56, in main swh-lister_1 | return swh(auto_envvar_prefix='SWH') swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/click/core.py", line 764, in __call__ swh-lister_1 | return self.main(*args, **kwargs) swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/click/core.py", line 717, in main swh-lister_1 | rv = self.invoke(ctx) swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/click/core.py", line 1137, in invoke swh-lister_1 | return _process_result(sub_ctx.command.invoke(sub_ctx)) swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/click/core.py", line 1137, in invoke swh-lister_1 | return _process_result(sub_ctx.command.invoke(sub_ctx)) swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/click/core.py", line 956, in invoke swh-lister_1 | return ctx.invoke(self.callback, **ctx.params) swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/click/core.py", line 555, in invoke swh-lister_1 | return callback(*args, **kwargs) swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/click/decorators.py", line 17, in new_func swh-lister_1 | return f(get_current_context(), *args, **kwargs) swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/lister/cli.py", line 155, in register_task_types swh-lister_1 | mod = import_module(task_module) swh-lister_1 | File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module swh-lister_1 | return _bootstrap._gcd_import(name[level:], package, level) swh-lister_1 | File "<frozen importlib._bootstrap>", line 1006, in _gcd_import swh-lister_1 | File "<frozen importlib._bootstrap>", line 983, in _find_and_load swh-lister_1 | File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked swh-lister_1 | File "<frozen importlib._bootstrap>", line 677, in _load_unlocked swh-lister_1 | File "<frozen importlib._bootstrap_external>", line 728, in exec_module swh-lister_1 | File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/lister/bitbucket/tasks.py", line 8, in <module> swh-lister_1 | from swh.scheduler.celery_backend.config import app swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/scheduler/celery_backend/config.py", line 191, in <module> swh-lister_1 | CONFIG = SWH_CONFIG.get('celery', default={}) swh-lister_1 | TypeError: get() takes no keyword argumentsI'll open a diff.