No OneTemporary
Actions

Size

39 KB

Subscribers

None

View Options

	diff --git a/conftest.py b/conftest.py
	new file mode 100644
	index 0000000..aeb1ec3
	--- /dev/null
	+++ b/conftest.py
	@@ -0,0 +1,6 @@
	+# Copyright (C) 2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	+pytest_plugins = ["swh.scheduler.pytest_plugin"]
	diff --git a/pytest.ini b/pytest.ini
	index afa4cf3..7cc9504 100644
	--- a/pytest.ini
	+++ b/pytest.ini
	@@ -1,2 +1,5 @@
	[pytest]
	+# We need this to avoid conflicts between the autoloaded fixture, and the
	+# manually loaded one.
	+addopts = -p no:pytest_swh_scheduler
	norecursedirs = docs
	diff --git a/requirements-test.txt b/requirements-test.txt
	index 7cc4593..833da5c 100644
	--- a/requirements-test.txt
	+++ b/requirements-test.txt
	@@ -1,5 +1,6 @@
	pytest
	pytest-postgresql
	requests_mock
	-testing.postgresql
	sqlalchemy-stubs
	+swh.scheduler[testing] >= 0.5
	+testing.postgresql
	diff --git a/swh/lister/bitbucket/tests/test_tasks.py b/swh/lister/bitbucket/tests/test_tasks.py
	index 9441f09..3f02b9a 100644
	--- a/swh/lister/bitbucket/tests/test_tasks.py
	+++ b/swh/lister/bitbucket/tests/test_tasks.py
	@@ -1,86 +1,95 @@
	+# Copyright (C) 2019-2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from time import sleep
	from celery.result import GroupResult

	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.bitbucket.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.bitbucket.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.bitbucket.tasks.BitBucketLister")
	-def test_incremental(lister, swh_app, celery_session_worker):
	+def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked BitbucketLister
	lister.return_value = lister
	lister.db_last_index.return_value = 42
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.bitbucket.tasks.IncrementalBitBucketLister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.bitbucket.tasks.IncrementalBitBucketLister"
	+ )
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_called_once_with()
	lister.run.assert_called_once_with(min_bound=42, max_bound=None)


	@patch("swh.lister.bitbucket.tasks.BitBucketLister")
	-def test_range(lister, swh_app, celery_session_worker):
	+def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked BitbucketLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task(
	+ res = swh_scheduler_celery_app.send_task(
	"swh.lister.bitbucket.tasks.RangeBitBucketLister", kwargs=dict(start=12, end=42)
	)
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with(min_bound=12, max_bound=42)


	@patch("swh.lister.bitbucket.tasks.BitBucketLister")
	-def test_relister(lister, swh_app, celery_session_worker):
	+def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked BitbucketLister
	lister.return_value = lister
	lister.run.return_value = None
	lister.db_partition_indices.return_value = [(i, i + 9) for i in range(0, 50, 10)]

	- res = swh_app.send_task("swh.lister.bitbucket.tasks.FullBitBucketRelister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.bitbucket.tasks.FullBitBucketRelister"
	+ )
	assert res

	res.wait()
	assert res.successful()

	# retrieve the GroupResult for this task and wait for all the subtasks
	# to complete
	promise_id = res.result
	assert promise_id
	- promise = GroupResult.restore(promise_id, app=swh_app)
	+ promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app)
	for i in range(5):
	if promise.ready():
	break
	sleep(1)

	lister.assert_called_with()

	# one by the FullBitbucketRelister task
	# + 5 for the RangeBitbucketLister subtasks
	assert lister.call_count == 6

	lister.db_last_index.assert_not_called()
	lister.db_partition_indices.assert_called_once_with(10000)

	# lister.run should have been called once per partition interval
	for i in range(5):
	assert (
	dict(min_bound=10 * i, max_bound=10 * i + 9),
	) in lister.run.call_args_list
	diff --git a/swh/lister/cgit/tests/test_tasks.py b/swh/lister/cgit/tests/test_tasks.py
	index 866bfde..c7d2fe3 100644
	--- a/swh/lister/cgit/tests/test_tasks.py
	+++ b/swh/lister/cgit/tests/test_tasks.py
	@@ -1,28 +1,33 @@
	+# Copyright (C) 2019-2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.cgit.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.cgit.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.cgit.tasks.CGitLister")
	-def test_lister(lister, swh_app, celery_session_worker):
	+def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked CGitLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task(
	+ res = swh_scheduler_celery_app.send_task(
	"swh.lister.cgit.tasks.CGitListerTask",
	kwargs=dict(url="https://git.kernel.org/", instance="kernel"),
	)
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with(url="https://git.kernel.org/", instance="kernel")
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with()
	diff --git a/swh/lister/core/tests/conftest.py b/swh/lister/core/tests/conftest.py
	index 5e62987..21ce76f 100644
	--- a/swh/lister/core/tests/conftest.py
	+++ b/swh/lister/core/tests/conftest.py
	@@ -1,53 +1,51 @@
	-# Copyright (C) 2019 The Software Heritage developers
	+# Copyright (C) 2019-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-from swh.scheduler.tests.conftest import * # noqa
	-
	import logging
	import pytest

	from sqlalchemy import create_engine

	from swh.lister import get_lister, SUPPORTED_LISTERS
	from swh.lister.core.models import initialize

	logger = logging.getLogger(__name__)


	@pytest.fixture
	def lister_db_url(postgresql_proc, postgresql):
	db_url = "postgresql://{user}@{host}:{port}/{dbname}".format(
	host=postgresql_proc.host,
	port=postgresql_proc.port,
	user="postgres",
	dbname="tests",
	)
	logger.debug("lister db_url: %s", db_url)
	return db_url


	@pytest.fixture
	def swh_listers(request, lister_db_url, swh_scheduler):

	listers = {}

	# Prepare schema for all listers
	for lister_name in SUPPORTED_LISTERS:
	lister = get_lister(lister_name, db_url=lister_db_url)
	lister.scheduler = swh_scheduler # inject scheduler fixture
	listers[lister_name] = lister
	initialize(create_engine(lister_db_url), drop_tables=True)

	# Add the load-archive-files expected by some listers (gnu, cran, ...)
	swh_scheduler.create_task_type(
	{
	"type": "load-archive-files",
	"description": "Load archive files.",
	"backend_name": "swh.loader.package.tasks.LoadArchive",
	"default_interval": "1 day",
	}
	)

	return listers
	diff --git a/swh/lister/cran/tests/test_tasks.py b/swh/lister/cran/tests/test_tasks.py
	index 9ff3d9f..8148a0a 100644
	--- a/swh/lister/cran/tests/test_tasks.py
	+++ b/swh/lister/cran/tests/test_tasks.py
	@@ -1,25 +1,30 @@
	+# Copyright (C) 2019-2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.cran.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.cran.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.cran.tasks.CRANLister")
	-def test_lister(lister, swh_app, celery_session_worker):
	+def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked CRANLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.cran.tasks.CRANListerTask")
	+ res = swh_scheduler_celery_app.send_task("swh.lister.cran.tasks.CRANListerTask")
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with()
	diff --git a/swh/lister/debian/tests/test_tasks.py b/swh/lister/debian/tests/test_tasks.py
	index 4fb08bd..dbf1136 100644
	--- a/swh/lister/debian/tests/test_tasks.py
	+++ b/swh/lister/debian/tests/test_tasks.py
	@@ -1,29 +1,31 @@
	-# Copyright (C) 2019 The Software Heritage developers
	+# Copyright (C) 2019-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.debian.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.debian.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.debian.tasks.DebianLister")
	-def test_lister(lister, swh_app, celery_session_worker):
	+def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked DebianLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.debian.tasks.DebianListerTask", ("stretch",))
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.debian.tasks.DebianListerTask", ("stretch",)
	+ )
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with(distribution="stretch")
	lister.run.assert_called_once_with()
	diff --git a/swh/lister/gitea/tests/test_tasks.py b/swh/lister/gitea/tests/test_tasks.py
	index 16352fa..4c223b6 100644
	--- a/swh/lister/gitea/tests/test_tasks.py
	+++ b/swh/lister/gitea/tests/test_tasks.py
	@@ -1,141 +1,150 @@
	+# Copyright (C) 2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from time import sleep
	from celery.result import GroupResult

	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.gitea.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.gitea.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.gitea.tasks.GiteaLister")
	-def test_incremental(lister, swh_app, celery_session_worker):
	+def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GiteaLister
	lister.return_value = lister
	lister.run.return_value = None
	lister.get_pages_information.return_value = (None, 10, None)

	- res = swh_app.send_task("swh.lister.gitea.tasks.IncrementalGiteaLister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.gitea.tasks.IncrementalGiteaLister"
	+ )
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with(sort="desc")
	lister.db_last_index.assert_not_called()
	lister.get_pages_information.assert_called_once_with()
	lister.run.assert_called_once_with(min_bound=1, max_bound=10, check_existence=True)


	@patch("swh.lister.gitea.tasks.GiteaLister")
	-def test_range(lister, swh_app, celery_session_worker):
	+def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GiteaLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task(
	+ res = swh_scheduler_celery_app.send_task(
	"swh.lister.gitea.tasks.RangeGiteaLister", kwargs=dict(start=12, end=42)
	)
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with(min_bound=12, max_bound=42)


	@patch("swh.lister.gitea.tasks.GiteaLister")
	-def test_relister(lister, swh_app, celery_session_worker):
	+def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GiteaLister
	lister.return_value = lister
	lister.run.return_value = None
	lister.get_pages_information.return_value = (None, 85, None)
	lister.db_partition_indices.return_value = [
	(i, i + 9) for i in range(0, 80, 10)
	] + [(80, 85)]

	- res = swh_app.send_task("swh.lister.gitea.tasks.FullGiteaRelister")
	+ res = swh_scheduler_celery_app.send_task("swh.lister.gitea.tasks.FullGiteaRelister")
	assert res

	res.wait()
	assert res.successful()

	# retrieve the GroupResult for this task and wait for all the subtasks
	# to complete
	promise_id = res.result
	assert promise_id
	- promise = GroupResult.restore(promise_id, app=swh_app)
	+ promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app)
	for i in range(5):
	if promise.ready():
	break
	sleep(1)

	lister.assert_called_with()

	# one by the FullGiteaRelister task
	# + 9 for the RangeGiteaLister subtasks
	assert lister.call_count == 10

	lister.db_last_index.assert_not_called()
	lister.db_partition_indices.assert_not_called()
	lister.get_pages_information.assert_called_once_with()

	# lister.run should have been called once per partition interval
	for i in range(8):
	# XXX inconsistent behavior: max_bound is EXCLUDED here
	assert (
	dict(min_bound=10 * i, max_bound=10 * i + 10),
	) in lister.run.call_args_list
	assert (dict(min_bound=80, max_bound=85),) in lister.run.call_args_list


	@patch("swh.lister.gitea.tasks.GiteaLister")
	-def test_relister_instance(lister, swh_app, celery_session_worker):
	+def test_relister_instance(
	+ lister, swh_scheduler_celery_app, swh_scheduler_celery_worker
	+):
	# setup the mocked GiteaLister
	lister.return_value = lister
	lister.run.return_value = None
	lister.get_pages_information.return_value = (None, 85, None)
	lister.db_partition_indices.return_value = [
	(i, i + 9) for i in range(0, 80, 10)
	] + [(80, 85)]

	- res = swh_app.send_task(
	+ res = swh_scheduler_celery_app.send_task(
	"swh.lister.gitea.tasks.FullGiteaRelister",
	kwargs=dict(url="https://0xacab.org/api/v4"),
	)
	assert res

	res.wait()
	assert res.successful()

	# retrieve the GroupResult for this task and wait for all the subtasks
	# to complete
	promise_id = res.result
	assert promise_id
	- promise = GroupResult.restore(promise_id, app=swh_app)
	+ promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app)
	for i in range(5):
	if promise.ready():
	break
	sleep(1)

	lister.assert_called_with(url="https://0xacab.org/api/v4")

	# one by the FullGiteaRelister task
	# + 9 for the RangeGiteaLister subtasks
	assert lister.call_count == 10

	lister.db_last_index.assert_not_called()
	lister.db_partition_indices.assert_not_called()
	lister.get_pages_information.assert_called_once_with()

	# lister.run should have been called once per partition interval
	for i in range(8):
	# XXX inconsistent behavior: max_bound is EXCLUDED here
	assert (
	dict(min_bound=10 * i, max_bound=10 * i + 10),
	) in lister.run.call_args_list
	assert (dict(min_bound=80, max_bound=85),) in lister.run.call_args_list
	diff --git a/swh/lister/github/tests/test_tasks.py b/swh/lister/github/tests/test_tasks.py
	index 721d88d..409ed6e 100644
	--- a/swh/lister/github/tests/test_tasks.py
	+++ b/swh/lister/github/tests/test_tasks.py
	@@ -1,87 +1,91 @@
	from time import sleep
	from celery.result import GroupResult

	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.github.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.github.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.github.tasks.GitHubLister")
	-def test_incremental(lister, swh_app, celery_session_worker):
	+def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GitHubLister
	lister.return_value = lister
	lister.db_last_index.return_value = 42
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.github.tasks.IncrementalGitHubLister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.github.tasks.IncrementalGitHubLister"
	+ )
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_called_once_with()
	lister.run.assert_called_once_with(min_bound=42, max_bound=None)


	@patch("swh.lister.github.tasks.GitHubLister")
	-def test_range(lister, swh_app, celery_session_worker):
	+def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GitHubLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task(
	+ res = swh_scheduler_celery_app.send_task(
	"swh.lister.github.tasks.RangeGitHubLister", kwargs=dict(start=12, end=42)
	)
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with(min_bound=12, max_bound=42)


	@patch("swh.lister.github.tasks.GitHubLister")
	-def test_relister(lister, swh_app, celery_session_worker):
	+def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GitHubLister
	lister.return_value = lister
	lister.run.return_value = None
	lister.db_partition_indices.return_value = [(i, i + 9) for i in range(0, 50, 10)]

	- res = swh_app.send_task("swh.lister.github.tasks.FullGitHubRelister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.github.tasks.FullGitHubRelister"
	+ )
	assert res

	res.wait()
	assert res.successful()

	# retrieve the GroupResult for this task and wait for all the subtasks
	# to complete
	promise_id = res.result
	assert promise_id
	- promise = GroupResult.restore(promise_id, app=swh_app)
	+ promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app)
	for i in range(5):
	if promise.ready():
	break
	sleep(1)

	lister.assert_called_with()

	# one by the FullGitHubRelister task
	# + 5 for the RangeGitHubLister subtasks
	assert lister.call_count == 6

	lister.db_last_index.assert_not_called()
	lister.db_partition_indices.assert_called_once_with(10000)

	# lister.run should have been called once per partition interval
	for i in range(5):
	# XXX inconsistent behavior: max_bound is INCLUDED here
	assert (
	dict(min_bound=10 * i, max_bound=10 * i + 9),
	) in lister.run.call_args_list
	diff --git a/swh/lister/gitlab/tests/test_tasks.py b/swh/lister/gitlab/tests/test_tasks.py
	index 1959989..1e7f27e 100644
	--- a/swh/lister/gitlab/tests/test_tasks.py
	+++ b/swh/lister/gitlab/tests/test_tasks.py
	@@ -1,141 +1,152 @@
	+# Copyright (C) 2019-2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from time import sleep
	from celery.result import GroupResult

	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.gitlab.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.gitlab.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.gitlab.tasks.GitLabLister")
	-def test_incremental(lister, swh_app, celery_session_worker):
	+def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GitlabLister
	lister.return_value = lister
	lister.run.return_value = None
	lister.get_pages_information.return_value = (None, 10, None)

	- res = swh_app.send_task("swh.lister.gitlab.tasks.IncrementalGitLabLister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.gitlab.tasks.IncrementalGitLabLister"
	+ )
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with(sort="desc")
	lister.db_last_index.assert_not_called()
	lister.get_pages_information.assert_called_once_with()
	lister.run.assert_called_once_with(min_bound=1, max_bound=10, check_existence=True)


	@patch("swh.lister.gitlab.tasks.GitLabLister")
	-def test_range(lister, swh_app, celery_session_worker):
	+def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GitlabLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task(
	+ res = swh_scheduler_celery_app.send_task(
	"swh.lister.gitlab.tasks.RangeGitLabLister", kwargs=dict(start=12, end=42)
	)
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with(min_bound=12, max_bound=42)


	@patch("swh.lister.gitlab.tasks.GitLabLister")
	-def test_relister(lister, swh_app, celery_session_worker):
	+def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GitlabLister
	lister.return_value = lister
	lister.run.return_value = None
	lister.get_pages_information.return_value = (None, 85, None)
	lister.db_partition_indices.return_value = [
	(i, i + 9) for i in range(0, 80, 10)
	] + [(80, 85)]

	- res = swh_app.send_task("swh.lister.gitlab.tasks.FullGitLabRelister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.gitlab.tasks.FullGitLabRelister"
	+ )
	assert res

	res.wait()
	assert res.successful()

	# retrieve the GroupResult for this task and wait for all the subtasks
	# to complete
	promise_id = res.result
	assert promise_id
	- promise = GroupResult.restore(promise_id, app=swh_app)
	+ promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app)
	for i in range(5):
	if promise.ready():
	break
	sleep(1)

	lister.assert_called_with()

	# one by the FullGitlabRelister task
	# + 9 for the RangeGitlabLister subtasks
	assert lister.call_count == 10

	lister.db_last_index.assert_not_called()
	lister.db_partition_indices.assert_not_called()
	lister.get_pages_information.assert_called_once_with()

	# lister.run should have been called once per partition interval
	for i in range(8):
	# XXX inconsistent behavior: max_bound is EXCLUDED here
	assert (
	dict(min_bound=10 * i, max_bound=10 * i + 10),
	) in lister.run.call_args_list
	assert (dict(min_bound=80, max_bound=85),) in lister.run.call_args_list


	@patch("swh.lister.gitlab.tasks.GitLabLister")
	-def test_relister_instance(lister, swh_app, celery_session_worker):
	+def test_relister_instance(
	+ lister, swh_scheduler_celery_app, swh_scheduler_celery_worker
	+):
	# setup the mocked GitlabLister
	lister.return_value = lister
	lister.run.return_value = None
	lister.get_pages_information.return_value = (None, 85, None)
	lister.db_partition_indices.return_value = [
	(i, i + 9) for i in range(0, 80, 10)
	] + [(80, 85)]

	- res = swh_app.send_task(
	+ res = swh_scheduler_celery_app.send_task(
	"swh.lister.gitlab.tasks.FullGitLabRelister",
	kwargs=dict(url="https://0xacab.org/api/v4"),
	)
	assert res

	res.wait()
	assert res.successful()

	# retrieve the GroupResult for this task and wait for all the subtasks
	# to complete
	promise_id = res.result
	assert promise_id
	- promise = GroupResult.restore(promise_id, app=swh_app)
	+ promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app)
	for i in range(5):
	if promise.ready():
	break
	sleep(1)

	lister.assert_called_with(url="https://0xacab.org/api/v4")

	# one by the FullGitlabRelister task
	# + 9 for the RangeGitlabLister subtasks
	assert lister.call_count == 10

	lister.db_last_index.assert_not_called()
	lister.db_partition_indices.assert_not_called()
	lister.get_pages_information.assert_called_once_with()

	# lister.run should have been called once per partition interval
	for i in range(8):
	# XXX inconsistent behavior: max_bound is EXCLUDED here
	assert (
	dict(min_bound=10 * i, max_bound=10 * i + 10),
	) in lister.run.call_args_list
	assert (dict(min_bound=80, max_bound=85),) in lister.run.call_args_list
	diff --git a/swh/lister/gnu/tests/test_tasks.py b/swh/lister/gnu/tests/test_tasks.py
	index d496798..d91f86a 100644
	--- a/swh/lister/gnu/tests/test_tasks.py
	+++ b/swh/lister/gnu/tests/test_tasks.py
	@@ -1,25 +1,30 @@
	+# Copyright (C) 2019-2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.gnu.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.gnu.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.gnu.tasks.GNULister")
	-def test_lister(lister, swh_app, celery_session_worker):
	+def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked GNULister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.gnu.tasks.GNUListerTask")
	+ res = swh_scheduler_celery_app.send_task("swh.lister.gnu.tasks.GNUListerTask")
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with()
	diff --git a/swh/lister/launchpad/tests/test_tasks.py b/swh/lister/launchpad/tests/test_tasks.py
	index ea676c5..4642373 100644
	--- a/swh/lister/launchpad/tests/test_tasks.py
	+++ b/swh/lister/launchpad/tests/test_tasks.py
	@@ -1,33 +1,42 @@
	+# Copyright (C) 2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from unittest.mock import patch


	@patch("swh.lister.launchpad.tasks.LaunchpadLister")
	-def test_new(lister, swh_app, celery_session_worker):
	+def test_new(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked LaunchpadLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.launchpad.tasks.NewLaunchpadLister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.launchpad.tasks.NewLaunchpadLister"
	+ )
	assert res
	res.wait()
	assert res.successful()

	assert lister.call_count == 2
	lister.db_last_threshold.assert_called_once()
	lister.run.assert_called_once()


	@patch("swh.lister.launchpad.tasks.LaunchpadLister")
	-def test_full(lister, swh_app, celery_session_worker):
	+def test_full(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked LaunchpadLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.launchpad.tasks.FullLaunchpadLister")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.launchpad.tasks.FullLaunchpadLister"
	+ )
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once()
	lister.db_last_threshold.assert_not_called()
	lister.run.assert_called_once_with(max_bound=None)
	diff --git a/swh/lister/npm/tests/test_tasks.py b/swh/lister/npm/tests/test_tasks.py
	index 382e557..481d91d 100644
	--- a/swh/lister/npm/tests/test_tasks.py
	+++ b/swh/lister/npm/tests/test_tasks.py
	@@ -1,52 +1,61 @@
	+# Copyright (C) 2019-2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from contextlib import contextmanager
	from unittest.mock import patch


	@contextmanager
	def mock_save(lister):
	yield


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.npm.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.npm.tasks.save_registry_state")
	@patch("swh.lister.npm.tasks.NpmLister")
	-def test_lister(lister, save, swh_app, celery_session_worker):
	+def test_lister(lister, save, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked NpmLister
	lister.return_value = lister
	lister.run.return_value = None
	save.side_effect = mock_save

	- res = swh_app.send_task("swh.lister.npm.tasks.NpmListerTask")
	+ res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.NpmListerTask")
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.run.assert_called_once_with()


	@patch("swh.lister.npm.tasks.save_registry_state")
	@patch("swh.lister.npm.tasks.get_last_update_seq")
	@patch("swh.lister.npm.tasks.NpmIncrementalLister")
	-def test_incremental(lister, seq, save, swh_app, celery_session_worker):
	+def test_incremental(
	+ lister, seq, save, swh_scheduler_celery_app, swh_scheduler_celery_worker
	+):
	# setup the mocked NpmLister
	lister.return_value = lister
	lister.run.return_value = None
	seq.return_value = 42
	save.side_effect = mock_save

	- res = swh_app.send_task("swh.lister.npm.tasks.NpmIncrementalListerTask")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.npm.tasks.NpmIncrementalListerTask"
	+ )
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	seq.assert_called_once_with(lister)
	lister.run.assert_called_once_with(min_bound=42)
	diff --git a/swh/lister/packagist/tests/test_tasks.py b/swh/lister/packagist/tests/test_tasks.py
	index 7c89b5b..6c5d15d 100644
	--- a/swh/lister/packagist/tests/test_tasks.py
	+++ b/swh/lister/packagist/tests/test_tasks.py
	@@ -1,29 +1,31 @@
	-# Copyright (C) 2019 the Software Heritage developers
	+# Copyright (C) 2019-2020 the Software Heritage developers
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.packagist.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.packagist.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.packagist.tasks.PackagistLister")
	-def test_lister(lister, swh_app, celery_session_worker):
	+def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked PackagistLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.packagist.tasks.PackagistListerTask")
	+ res = swh_scheduler_celery_app.send_task(
	+ "swh.lister.packagist.tasks.PackagistListerTask"
	+ )
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with()
	diff --git a/swh/lister/phabricator/tests/test_tasks.py b/swh/lister/phabricator/tests/test_tasks.py
	index 38e1686..1aca578 100644
	--- a/swh/lister/phabricator/tests/test_tasks.py
	+++ b/swh/lister/phabricator/tests/test_tasks.py
	@@ -1,11 +1,11 @@
	-# Copyright (C) 2019 the Software Heritage developers
	+# Copyright (C) 2019-2020 the Software Heritage developers
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.phabricator.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.phabricator.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"
	diff --git a/swh/lister/pypi/tests/test_tasks.py b/swh/lister/pypi/tests/test_tasks.py
	index 89ffeac..2443cce 100644
	--- a/swh/lister/pypi/tests/test_tasks.py
	+++ b/swh/lister/pypi/tests/test_tasks.py
	@@ -1,25 +1,30 @@
	+# Copyright (C) 2019-2020 The Software Heritage developers
	+# See the AUTHORS file at the top-level directory of this distribution
	+# License: GNU General Public License version 3, or any later version
	+# See top-level LICENSE file for more information
	+
	from unittest.mock import patch


	-def test_ping(swh_app, celery_session_worker):
	- res = swh_app.send_task("swh.lister.pypi.tasks.ping")
	+def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	+ res = swh_scheduler_celery_app.send_task("swh.lister.pypi.tasks.ping")
	assert res
	res.wait()
	assert res.successful()
	assert res.result == "OK"


	@patch("swh.lister.pypi.tasks.PyPILister")
	-def test_lister(lister, swh_app, celery_session_worker):
	+def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	# setup the mocked PypiLister
	lister.return_value = lister
	lister.run.return_value = None

	- res = swh_app.send_task("swh.lister.pypi.tasks.PyPIListerTask")
	+ res = swh_scheduler_celery_app.send_task("swh.lister.pypi.tasks.PyPIListerTask")
	assert res
	res.wait()
	assert res.successful()

	lister.assert_called_once_with()
	lister.db_last_index.assert_not_called()
	lister.run.assert_called_once_with()
	diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py
	index 4ac7b1b..a333804 100644
	--- a/swh/lister/tests/test_cli.py
	+++ b/swh/lister/tests/test_cli.py
	@@ -1,66 +1,66 @@
	-# Copyright (C) 2019 The Software Heritage developers
	+# Copyright (C) 2019-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import pytest


	from swh.lister.core.lister_base import ListerBase
	from swh.lister.cli import get_lister, SUPPORTED_LISTERS

	from .test_utils import init_db


	def test_get_lister_wrong_input():
	"""Unsupported lister should raise"""
	with pytest.raises(ValueError) as e:
	get_lister("unknown", "db-url")

	assert "Invalid lister" in str(e.value)


	def test_get_lister():
	"""Instantiating a supported lister should be ok

	"""
	db_url = init_db().url()
	for lister_name in SUPPORTED_LISTERS:
	lst = get_lister(lister_name, db_url)
	assert isinstance(lst, ListerBase)


	def test_get_lister_override():
	"""Overriding the lister configuration should populate its config

	"""
	db_url = init_db().url()

	listers = {
	"gitlab": "https://other.gitlab.uni/api/v4/",
	"phabricator": "https://somewhere.org/api/diffusion.repository.search",
	"cgit": "https://some.where/cgit",
	}

	# check the override ends up defined in the lister
	for lister_name, url in listers.items():
	lst = get_lister(
	lister_name,
	db_url,
	**{"url": url, "priority": "high", "policy": "oneshot",},
	)

	assert lst.url == url
	assert lst.config["priority"] == "high"
	assert lst.config["policy"] == "oneshot"

	# check the default urls are used and not the override (since it's not
	# passed)
	for lister_name, url in listers.items():
	lst = get_lister(lister_name, db_url)

	# no override so this does not end up in lister's configuration
	assert "url" not in lst.config
	assert "priority" not in lst.config
	assert "oneshot" not in lst.config
	assert lst.url == lst.DEFAULT_URL
	diff --git a/tox.ini b/tox.ini
	index ef77275..ff5b5ec 100644
	--- a/tox.ini
	+++ b/tox.ini
	@@ -1,36 +1,37 @@
	[tox]
	envlist=black,flake8,mypy,py3

	[testenv]
	extras =
	testing
	deps =
	swh.core[http] >= 0.0.61
	+ swh.scheduler[testing] >= 0.5.0
	pytest-cov
	dev: ipdb
	commands =
	pytest \
	!dev: --cov={envsitepackagesdir}/swh/lister/ --cov-branch \
	{envsitepackagesdir}/swh/lister/ {posargs}

	[testenv:black]
	skip_install = true
	deps =
	black
	commands =
	{envpython} -m black --check swh

	[testenv:flake8]
	skip_install = true
	deps =
	flake8
	commands =
	{envpython} -m flake8

	[testenv:mypy]
	extras =
	testing
	deps =
	mypy
	commands =
	mypy swh

File Metadata

Mime Type: text/x-diff
Expires: Thu, Jul 3, 10:42 AM (2 w, 12 h ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3273704

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions