diff --git a/README.md b/README.md index 3673eaf..01a2c41 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,45 @@ swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). # Tests ## Running test manually ### Test data To be able to run (unit) tests, you need to have the [[https://forge.softwareheritage.org/source/swh-storage-testdata.git|swh-storage-testdata]] in the parent directory. If you have set your environment following the [[ https://docs.softwareheritage.org/devel/getting-started.html#getting-started|Getting started]] document everythong should be set up just fine. Otherwise: ``` ~/.../swh-scheduler$ git clone https://forge.softwareheritage.org/source/swh-storage-testdata.git ../swh-storage-testdata ``` ### Required services -Unit tests may require a running celery broker on your system (rabbitmq by -default). You can set the `BROKER_URL` environment variable to specify the url -to be used. - -If you do not want to use your system's broker (or do not want to have one -running), you shold use [[ https://github.com/jd/pifpaf | pifpaf ]] to take -care of that for you. - +Unit tests that require a running celery broker uses an in memory broker/result +backend by default, but you can choose to use a true broker by setting +`CELERY_BROKER_URL` and `CELERY_RESULT_BACKEND` environment variables up. For example: ``` -$ pifpaf --env-prefix PG run postgresql -- \ - pifpaf --env-prefix AMQP run rabbitmq nosetests +$ CELERY_BROKER_URL=amqp://localhost pifpaf run postgresql nosetests ..................................... ---------------------------------------------------------------------- Ran 37 tests in 15.578s OK ``` diff --git a/swh/scheduler/tests/celery_testing.py b/swh/scheduler/tests/celery_testing.py index 5f0ab61..2859e9c 100644 --- a/swh/scheduler/tests/celery_testing.py +++ b/swh/scheduler/tests/celery_testing.py @@ -1,37 +1,18 @@ import os -import urllib - -from celery import current_app - - -CELERY_BROKER_PROTOCOLS = ['amqp', 'redis', 'sqs'] def setup_celery(): - broker_url = None - if 'BROKER_URL' in os.environ: - broker_url = os.environ['BROKER_URL'] - - elif 'PIFPAF_URLS' in os.environ: - urls = os.environ['PIFPAF_URLS'].split(';') - urls = [x.replace('rabbit://', 'amqp://') for x in urls] - for url in urls: - scheme = urllib.parse.splittype(url)[0] - if scheme in CELERY_BROKER_PROTOCOLS: - broker_url = url - break - if broker_url: - current_app.conf.broker_url = broker_url + os.environ.setdefault('CELERY_BROKER_URL', 'memory://') + os.environ.setdefault('CELERY_RESULT_BACKEND', 'cache+memory://') class CeleryTestFixture: - """Mix this in a test subject class to get Celery testing support configured - via environment variables, typically set up by pifpaf. + """Mix this in a test subject class to setup Celery config for testing + purpose. - It expect a connection url to a celery broker either as a BROKER_URL or a - url listed in a PIFPAF_URL environment variable. + Can be overriden by CELERY_BROKER_URL and CELERY_RESULT_BACKEND env vars. """ def setUp(sel): setup_celery() super().setUp()