Differential D1503 Diff 4945 swh/scheduler/celery_backend/config.py

Changeset View

Standalone View

swh/scheduler/celery_backend/config.py

# Copyright (C) 2015-2019 The Software Heritage developers		# Copyright (C) 2015-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import logging		import logging
import os		import os
		import pkg_resources
import urllib.parse		import urllib.parse

from celery import Celery		from celery import Celery
from celery.signals import setup_logging, celeryd_after_setup		from celery.signals import setup_logging, celeryd_after_setup
from celery.utils.log import ColorFormatter		from celery.utils.log import ColorFormatter
from celery.worker.control import Panel		from celery.worker.control import Panel

from kombu import Exchange, Queue		from kombu import Exchange, Queue
▲ Show 20 Lines • Show All 176 Lines • ▼ Show 20 Lines	if not CONFIG:
if INSTANCE_NAME:		if INSTANCE_NAME:
CONFIG_NAME = CONFIG_NAME_TEMPLATE % INSTANCE_NAME		CONFIG_NAME = CONFIG_NAME_TEMPLATE % INSTANCE_NAME
else:		else:
CONFIG_NAME = DEFAULT_CONFIG_NAME		CONFIG_NAME = DEFAULT_CONFIG_NAME

# Load the Celery config		# Load the Celery config
CONFIG = load_named_config(CONFIG_NAME, DEFAULT_CONFIG)		CONFIG = load_named_config(CONFIG_NAME, DEFAULT_CONFIG)

		CONFIG.setdefault('task_modules', [])
		# load tasks modules declared as plugin entry points
		for entrypoint in pkg_resources.iter_entry_points('swh.workers'):
		CONFIG['task_modules'].append(entrypoint.load()()['tasks'])
		ardumontUnsubmitted Not Done Inline Actions I have a little trouble reading that. Can you explicit what the `entrypoint.load()` returns... ... plugging brain cables... Nevermind, i think i grokked it: `entrypoint.load()` returns the function registered in the `setup.py`. For example, you dubbed it `swh.lister..__init__.register` in the other diff. and then you call it, to actually access the dict the `register` function returns Ok. Maybe this would be more readable? for ...: worker_register_fn = entrypoint.load() CONFIG['task_modules'].append(worker_register_fn()['tasks]) ardumont:* I have a little trouble reading that. Can you explicit what the `entrypoint.load()` returns...
		douarddaAuthorUnsubmitted Done Inline Actions a more explicit version is fine for me, will do douardda: a more explicit version is fine for me, will do

# Celery Queues		# Celery Queues
CELERY_QUEUES = [Queue('celery', Exchange('celery'), routing_key='celery')]		CELERY_QUEUES = [Queue('celery', Exchange('celery'), routing_key='celery')]

CELERY_DEFAULT_CONFIG = dict(		CELERY_DEFAULT_CONFIG = dict(
# Timezone configuration: all in UTC		# Timezone configuration: all in UTC
enable_utc=True,		enable_utc=True,
timezone='UTC',		timezone='UTC',
# Imported modules		# Imported modules
▲ Show 20 Lines • Show All 64 Lines • Show Last 20 Lines