diff --git a/debian/changelog b/debian/changelog new file mode 100644 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,676 @@ +swh-lister (0.5.4-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.5.4 - (tagged by Antoine R. Dumont + (@ardumont) on 2021-01-28 11:23:29 + +0100) + * Upstream changes: - v0.5.4 - gitlab: Deal with missing or + trailing / in url input - tox.ini: Work around build failure due + to upstream release + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 28 Jan 2021 10:27:59 +0000 + +swh-lister (0.5.2-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.5.2 - (tagged by Antoine R. Dumont + (@ardumont) on 2021-01-27 17:19:10 + +0100) + * Upstream changes: - v0.5.2 - test_cli: Drop launchpad lister + from the test_get_lister + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 27 Jan 2021 16:25:31 +0000 + +swh-lister (0.5.1-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.5.1 - (tagged by Antoine R. Dumont + (@ardumont) on 2021-01-27 16:39:20 + +0100) + * Upstream changes: - v0.5.1 - launchpad: Actually mock the + anonymous login to launchpad - Drop no longer + swh.lister.core.{indexing,page_by_page}_lister - tests: Drop + unneeded reset instruction - cgit: Don't stop the listing when a + repository page is not available + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 27 Jan 2021 15:47:39 +0000 + +swh-lister (0.5.0-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.5.0 - (tagged by Antoine R. Dumont + (@ardumont) on 2021-01-27 14:33:24 + +0100) + * Upstream changes: - v0.5.0 - cgit: Add support for + last_update information during listing - Port Debian lister to + new lister api - gitlab: Implement keyset-based pagination + listing - cran: Retrieve last update date for each listed + package - Port CRAN lister to new lister api - gitlab: Add + support for last_update information during listing - Port Gitea + lister to new lister api - Port cgit lister to the new lister + api - bitbucket: Pick random credentials in configuration and + improve logging - Port Gitlab lister to the new lister api - + Port Npm lister to new lister api - Port PyPI lister to new + lister api - Port Bitbucket lister to new lister api - Port + Phabricator lister to new lister api - Port GitHub lister to new + lister api - Introduce a simpler base pattern for lister + implementations + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 27 Jan 2021 13:40:34 +0000 + +swh-lister (0.4.0-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.4.0 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-11-23 15:47:05 + +0100) + * Upstream changes: - v0.4.0 - requirements: Rework + dependencies - tests: Reduce db initialization fixtures to a + minimum - Create listing task with a default of 3 if unspecified + - lister.pytest_plugin: Simplify fixture setup - tests: Clarify + listers test configuration + + -- Software Heritage autobuilder (on jenkins-debian1) Mon, 23 Nov 2020 14:52:03 +0000 + +swh-lister (0.3.0-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.3.0 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-10-19 09:50:43 + +0200) + * Upstream changes: - v0.3.0 - lister.config: Adapt scheduler + configuration structure - drop mock_get_scheduler which creates + indirection for no good reason + + -- Software Heritage autobuilder (on jenkins-debian1) Mon, 19 Oct 2020 07:56:17 +0000 + +swh-lister (0.2.1-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.2.1 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-10-07 14:02:42 + +0200) + * Upstream changes: - v0.2.1 - lister_base: Drop leftover + mixin SWHConfig which is no longer used + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 07 Oct 2020 12:07:43 +0000 + +swh-lister (0.2.0-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.2.0 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-10-06 09:33:33 + +0200) + * Upstream changes: - v0.2.0 - lister*: Migrate away from + SWHConfig mixin - tox.ini: pin black to the pre-commit version + (19.10b0) to avoid flip-flops - Run isort after the CLI import + changes + + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 06 Oct 2020 07:36:07 +0000 + +swh-lister (0.1.5-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.1.5 - (tagged by David Douard + on 2020-09-25 11:51:57 +0200) + * Upstream changes: - v0.1.5 + + -- Software Heritage autobuilder (on jenkins-debian1) Fri, 25 Sep 2020 09:55:44 +0000 + +swh-lister (0.1.4-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.1.4 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-09-10 11:32:46 + +0200) + * Upstream changes: - v0.1.4 - gitea.lister: Fix uid to be + unique across instance - utils.split_range: Split into not + overlapping ranges - gitea.tasks: Fix parameter name from 'sort' + to 'order' + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 10 Sep 2020 09:35:53 +0000 + +swh-lister (0.1.3-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.1.3 - (tagged by Vincent SELLIER + on 2020-09-08 14:48:08 +0200) + * Upstream changes: - v0.1.3 - Launchpad: rename task name to + match conventions - tests: Separate lister instantiations + + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Sep 2020 12:53:22 +0000 + +swh-lister (0.1.2-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.1.2 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-09-02 13:07:30 + +0200) + * Upstream changes: - v0.1.2 - pytest_plugin: Instantiate only + lister with no particular setup - pytest: Define plugin and + declare it in the root conftest + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 02 Sep 2020 11:10:14 +0000 + +swh-lister (0.1.1-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.1.1 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-09-01 16:08:48 + +0200) + * Upstream changes: - v0.1.1 - test_cli: Exclude launchpad + lister from the check + + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 01 Sep 2020 14:11:46 +0000 + +swh-lister (0.1.0-1~swh2) unstable-swh; urgency=medium + + * Update dependencies + + -- Antoine R. Dumont (@ardumont) Wed, 26 Aug 2020 16:05:03 +0000 + +swh-lister (0.1.0-1~swh1) unstable-swh; urgency=medium + + [ Nicolas Dandrimont ] + * Use setuptools-scm instead of vcversioner + + [ Software Heritage autobuilder (on jenkins-debian1) ] + * New upstream release 0.1.0 - (tagged by David Douard + on 2020-08-25 18:33:55 +0200) + * Upstream changes: - v0.1.0 + + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 25 Aug 2020 16:39:28 +0000 + +swh-lister (0.0.50-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.50 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-01-20 10:44:57 + +0100) + * Upstream changes: - v0.0.50 - github.lister: Filter out + partial repositories which break listing - docs: Fix sphinx + warnings - core.lister_base: Improve slightly docs and types + + -- Software Heritage autobuilder (on jenkins-debian1) Mon, 20 Jan 2020 09:51:23 +0000 + +swh-lister (0.0.49-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.49 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-01-17 14:20:35 + +0100) + * Upstream changes: - v0.0.49 - github.lister: Use Retry-After + header when rate limit reached + + -- Software Heritage autobuilder (on jenkins-debian1) Fri, 17 Jan 2020 13:27:56 +0000 + +swh-lister (0.0.48-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.48 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-01-16 13:56:12 + +0100) + * Upstream changes: - v0.0.48 - cran.lister: Use cran's + canonical url for origin url - cran.lister: Version uid so we + can list new package versions - cran.lister: Adapt docstring + sample accordingly + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 16 Jan 2020 13:03:54 +0000 + +swh-lister (0.0.47-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.47 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-01-09 10:26:18 + +0100) + * Upstream changes: - v0.0.47 - cran.lister: Align loading + tasks' with loader's expectation + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 09 Jan 2020 09:34:26 +0000 + +swh-lister (0.0.46-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.46 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-12-19 14:09:45 + +0100) + * Upstream changes: - v0.0.46 - lister.debian: Make debian + init step idempotent and up-to-date - lister_base: Split into + chunks the tasks prior to creation + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 19 Dec 2019 13:16:45 +0000 + +swh-lister (0.0.45-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.45 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-12-10 11:27:17 + +0100) + * Upstream changes: - v0.0.45 - core: Align listers' task + output (hg/git tasks) with expected format - npm: Align lister's + loader output tasks with expected format - lister/tasks: + Standardize return statements + + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 10 Dec 2019 10:32:45 +0000 + +swh-lister (0.0.44-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.44 - (tagged by Nicolas Dandrimont + on 2019-11-22 16:15:54 +0100) + * Upstream changes: - Release swh.lister v0.0.44 - Define + proper User Agents everywhere + + -- Software Heritage autobuilder (on jenkins-debian1) Fri, 22 Nov 2019 15:31:33 +0000 + +swh-lister (0.0.43-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.43 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-11-21 18:46:35 + +0100) + * Upstream changes: - v0.0.43 - lister.pypi: Align lister with + pypi package loader - lister.npm: Align lister with npm package + loader - lister.tests: Avoid duplication setup step - Fix + typos (and trailing ws) reported by codespell - Add a pre-commit + config file + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 21 Nov 2019 17:56:34 +0000 + +swh-lister (0.0.42-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.42 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-11-21 13:52:16 + +0100) + * Upstream changes: - v0.0.42 - cran/gnu: Rename task_type to + load-archive-files - lister.tests: Add missing task_type for + package listers - Migrate tox.ini to extras = xxx instead of + deps = .[testing] - Merge tox environments - Include all + requirements in MANIFEST.in - lister.cli: Remove task type + register cli + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 21 Nov 2019 13:00:29 +0000 + +swh-lister (0.0.41-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.41 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-11-15 12:02:13 + +0100) + * Upstream changes: - v0.0.41 - simple_lister: Flush to db + more frequently - gnu.lister: Use url as primary key - + gnu.lister.tests: Add missing assertion - gnu.lister: Add + missing retries_left parameter - debian.models: Migrate tests + from storage to debian lister model + + -- Software Heritage autobuilder (on jenkins-debian1) Fri, 15 Nov 2019 11:06:35 +0000 + +swh-lister (0.0.40-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.40 - (tagged by Nicolas Dandrimont + on 2019-11-13 13:54:38 +0100) + * Upstream changes: - Release swh.lister 0.0.40 - Fix bogus + NotImplementedError on Area.index_uris + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 13 Nov 2019 13:02:08 +0000 + +swh-lister (0.0.39-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.39 - (tagged by Nicolas Dandrimont + on 2019-11-13 13:23:31 +0100) + * Upstream changes: - Release swh.lister 0.0.39 - Properly + register all tasks - Fix up db_partition_indices to avoid + expensive scans + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 13 Nov 2019 12:28:33 +0000 + +swh-lister (0.0.38-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.38 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-11-06 15:55:46 + +0100) + * Upstream changes: - v0.0.38 - Remove swh.storage.schemata + remnants + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 15:00:16 +0000 + +swh-lister (0.0.37-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.37 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-11-06 15:06:51 + +0100) + * Upstream changes: - v0.0.37 - Update swh-core dependency + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 14:18:31 +0000 + +swh-lister (0.0.36-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.36 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-11-06 11:33:33 + +0100) + * Upstream changes: - v0.0.36 - lister.*.tests: Add at least + one integration test - gnu.lister: Move gnu listers specifity + within the lister's scope - debian/lister: Use url parameter + name instead of origin - debian/model: Install lister model + within the lister repository - lister.*.tasks: Stop binding + tasks to a specific instance of the - celery app - + cran.lister: Refactor and fix cran lister - github/lister: + Prevent erroneous scheduler tasks disabling - + phabricator/lister: Fix lister - setup.py: Kill deprecated swh- + lister command - Bootstrap typing annotations + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 06 Nov 2019 10:55:41 +0000 + +swh-lister (0.0.35-1~swh4) unstable-swh; urgency=medium + + * Fix runtime dependencies + + -- Antoine R. Dumont (@ardumont) Wed, 11 Sep 2019 10:58:01 +0200 + +swh-lister (0.0.35-1~swh3) unstable-swh; urgency=medium + + * Bump dh-python to >= 3 for pybuild.testfiles. + + -- Nicolas Dandrimont Tue, 10 Sep 2019 14:58:11 +0200 + +swh-lister (0.0.35-1~swh2) unstable-swh; urgency=medium + + * Add egg-info to pybuild.testfiles. Close T1995. + + -- Nicolas Dandrimont Tue, 10 Sep 2019 14:36:22 +0200 + +swh-lister (0.0.35-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.35 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-09-09 12:14:42 + +0200) + * Upstream changes: - v0.0.35 - Fix debian package + + -- Software Heritage autobuilder (on jenkins-debian1) Mon, 09 Sep 2019 10:19:02 +0000 + +swh-lister (0.0.34-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.34 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-09-06 14:03:39 + +0200) + * Upstream changes: - v0.0.34 - listers: Implement listers as + plugins - cgit: rewrite the CGit lister (and add more tests) + - listers: simplify and unify constructor use - phabricator: + randomly select the API token in the provided list - docs: Fix + toc + + -- Software Heritage autobuilder (on jenkins-debian1) Fri, 06 Sep 2019 12:09:13 +0000 + +swh-lister (0.0.33-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.33 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-08-29 10:23:20 + +0200) + * Upstream changes: - v0.0.33 - lister.cli: Allow to list + forges with policy and priority - listers: Add New packagist + lister - listers: Allow to override policy and priority for + scheduled tasks - tests: Add tests to cli, pypi and improve + lister core's - docs: Add code of conduct document + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 29 Aug 2019 08:28:23 +0000 + +swh-lister (0.0.32-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.32 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-28 18:21:50 + +0200) + * Upstream changes: - v0.0.32 - Clean up dead code - Add + missing *.html sample for tests to run in packaging + + -- Software Heritage autobuilder (on jenkins-debian1) Fri, 28 Jun 2019 16:42:05 +0000 + +swh-lister (0.0.31-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.31 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-28 17:57:48 + +0200) + * Upstream changes: - v0.0.31 - Add cgit instance lister - + Add back description in cran lister - Update contributors + + -- Software Heritage autobuilder (on jenkins-debian1) Fri, 28 Jun 2019 16:06:25 +0000 + +swh-lister (0.0.30-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.30 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-26 14:52:13 + +0200) + * Upstream changes: - v0.0.30 - Drop last description mentions + for gitlab and cran listers. + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 26 Jun 2019 13:02:11 +0000 + +swh-lister (0.0.29-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.29 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-26 12:37:14 + +0200) + * Upstream changes: - v0.0.29 - lister: Fix bitbucket lister + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 26 Jun 2019 10:47:20 +0000 + +swh-lister (0.0.28-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.28 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-20 12:00:09 + +0200) + * Upstream changes: - v0.0.28 - listers: Remove unused columns + `origin_id` / `description` - gnu-lister: Use origin-type as + 'tar' (and not 'gnu') - phabricator: Remove unused code + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 20 Jun 2019 10:07:48 +0000 + +swh-lister (0.0.27-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.27 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-18 10:27:09 + +0200) + * Upstream changes: - v0.0.27 - Unify lister tablenames to use + consistently singular - Add missing instance field to + phabricator repository model + + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Jun 2019 08:44:38 +0000 + +swh-lister (0.0.26-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.26 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-17 17:53:33 + +0200) + * Upstream changes: - v0.0.26 - phabricator.lister: Use + credentials setup from configuration file - gitlab.lister: + Remove request_params method override + + -- Software Heritage autobuilder (on jenkins-debian1) Mon, 17 Jun 2019 16:05:05 +0000 + +swh-lister (0.0.25-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.25 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-13 15:54:42 + +0200) + * Upstream changes: - v0.0.25 - Add new cran lister - + listers: Stop creating origins when scheduling new tasks + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 13 Jun 2019 13:59:30 +0000 + +swh-lister (0.0.24-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.24 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-06-12 12:02:54 + +0200) + * Upstream changes: - v0.0.24 - swh.lister.gnu: Add new gnu + lister + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 12 Jun 2019 10:10:56 +0000 + +swh-lister (0.0.23-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.23 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-05-29 14:04:22 + +0200) + * Upstream changes: - v0.0.23 - lister: Unify credentials + structure between listers + + -- Software Heritage autobuilder (on jenkins-debian1) Wed, 29 May 2019 12:10:51 +0000 + +swh-lister (0.0.22-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.22 - (tagged by Antoine Lambert + on 2019-05-23 10:59:39 +0200) + * Upstream changes: - version 0.0.22 + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 23 May 2019 09:05:34 +0000 + +swh-lister (0.0.21-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.21 - (tagged by Antoine Lambert + on 2019-04-11 11:00:55 +0200) + * Upstream changes: - version 0.0.21 + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 11 Apr 2019 09:05:30 +0000 + +swh-lister (0.0.20-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.20 - (tagged by Antoine R. Dumont + (@ardumont) on 2019-02-14 10:50:06 + +0100) + * Upstream changes: - v0.0.20 - d/*: debian packaging files + migrated to separated branches - lister.cli: Fix spelling typo + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 14 Feb 2019 09:59:29 +0000 + +swh-lister (0.0.19-1~swh1) unstable-swh; urgency=medium + + * New upstream release 0.0.19 - (tagged by David Douard + on 2019-02-07 17:36:33 +0100) + * Upstream changes: - v0.0.19 + + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 07 Feb 2019 16:42:39 +0000 + +swh-lister (0.0.18-1~swh1) unstable-swh; urgency=medium + + * v0.0.18 + * docs: add title and brief module description + * gitlab.lister: Break asap when problem exists during fetch info + * gitlab.lister: Do not expect gitlab instances to have credentials + * setup: prepare for pypi upload + * gitlab/models.py: drop unused import + + -- Antoine R. Dumont (@ardumont) Mon, 08 Oct 2018 15:54:12 +0200 + +swh-lister (0.0.17-1~swh1) unstable-swh; urgency=medium + + * v0.0.17 + * Change pypi project url to use the /project api + + -- Antoine R. Dumont (@ardumont) Tue, 18 Sep 2018 11:35:25 +0200 + +swh-lister (0.0.16-1~swh1) unstable-swh; urgency=medium + + * v0.0.16 + * Normalize PyPI name + + -- Antoine R. Dumont (@ardumont) Fri, 14 Sep 2018 13:25:56 +0200 + +swh-lister (0.0.15-1~swh1) unstable-swh; urgency=medium + + * v0.0.15 + * Add pypi lister + + -- Antoine R. Dumont (@ardumont) Thu, 06 Sep 2018 17:09:25 +0200 + +swh-lister (0.0.14-1~swh1) unstable-swh; urgency=medium + + * v0.0.14 + * core.lister_base: Batch create origins (storage) & tasks (scheduler) + * swh.lister.cli: Add debian lister to the list of supported listers + * README.md: Update to demo the lister debian run + + -- Antoine R. Dumont (@ardumont) Tue, 31 Jul 2018 15:46:12 +0200 + +swh-lister (0.0.13-1~swh1) unstable-swh; urgency=medium + + * v0.0.13 + * Fix missing use cases when unable to retrieve information from the + api + * server + * gitlab/lister: Allow specifying the number of elements to + * read (default is 20, same as the current gitlab api) + + -- Antoine R. Dumont (@ardumont) Fri, 20 Jul 2018 13:46:04 +0200 + +swh-lister (0.0.12-1~swh1) unstable-swh; urgency=medium + + * v0.0.12 + * swh.lister.gitlab.tasks: Use gitlab as instance name for gitlab.com + * README.md: Add gitlab to the lister implementations referenced + * core/lister_base: Remove unused import + + -- Antoine R. Dumont (@ardumont) Thu, 19 Jul 2018 11:29:14 +0200 + +swh-lister (0.0.11-1~swh1) unstable-swh; urgency=medium + + * v0.0.11 + * lister/gitlab: Add gitlab lister + * docs: Update documentation to demonstrate how to run a lister + locally + * core/lister: Make the listers' scheduler configuration adaptable + * debian/*: Fix debian packaging tests + + -- Antoine R. Dumont (@ardumont) Wed, 18 Jul 2018 14:16:56 +0200 + +swh-lister (0.0.10-1~swh1) unstable-swh; urgency=medium + + * Release swh.lister v0.0.10 + * Add missing task_queue attribute for debian listing tasks + * Make sure tests run during build + * Clean up runtime dependencies + + -- Nicolas Dandrimont Mon, 30 Oct 2017 17:37:25 +0100 + +swh-lister (0.0.9-1~swh1) unstable-swh; urgency=medium + + * Release swh.lister v0.0.9 + * Add tasks for the Debian lister + + -- Nicolas Dandrimont Mon, 30 Oct 2017 14:20:58 +0100 + +swh-lister (0.0.8-1~swh1) unstable-swh; urgency=medium + + * Release swh.lister v0.0.8 + * Add versioned dependency on sqlalchemy + + -- Nicolas Dandrimont Fri, 13 Oct 2017 12:15:38 +0200 + +swh-lister (0.0.7-1~swh1) unstable-swh; urgency=medium + + * Release swh.lister version 0.0.7 + * Update packaging runes + + -- Nicolas Dandrimont Thu, 12 Oct 2017 18:07:52 +0200 + +swh-lister (0.0.6-1~swh1) unstable-swh; urgency=medium + + * Release swh.lister v0.0.6 + * Add new debian lister + + -- Nicolas Dandrimont Wed, 11 Oct 2017 17:59:47 +0200 + +swh-lister (0.0.5-1~swh1) unstable-swh; urgency=medium + + * Release swh.lister 0.0.5 + * Make the lister more generic + * Add bitbucket lister + * Update tasks to new swh.scheduler API + + -- Nicolas Dandrimont Mon, 12 Jun 2017 18:22:13 +0200 + +swh-lister (0.0.4-1~swh1) unstable-swh; urgency=medium + + * v0.0.4 + * Update storage configuration reading + + -- Antoine R. Dumont (@ardumont) Thu, 15 Dec 2016 19:07:24 +0100 + +swh-lister (0.0.3-1~swh1) unstable-swh; urgency=medium + + * Release swh.lister.github v0.0.3 + * Generate swh.scheduler tasks and swh.storage origins on the fly + * Use celery tasks to schedule own work + + -- Nicolas Dandrimont Thu, 20 Oct 2016 17:30:39 +0200 + +swh-lister (0.0.2-1~swh1) unstable-swh; urgency=medium + + * Release swh.lister.github 0.0.2 + * Move constants to a constants module to avoid circular imports + + -- Nicolas Dandrimont Thu, 17 Mar 2016 20:35:11 +0100 + +swh-lister (0.0.1-1~swh1) unstable-swh; urgency=medium + + * Initial release + * Release swh.lister.github v0.0.1 + + -- Nicolas Dandrimont Thu, 17 Mar 2016 19:01:20 +0100 diff --git a/debian/clean b/debian/clean new file mode 100644 --- /dev/null +++ b/debian/clean @@ -0,0 +1 @@ +swh/lister/_version.py diff --git a/debian/compat b/debian/compat new file mode 100644 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 --- /dev/null +++ b/debian/control @@ -0,0 +1,41 @@ +Source: swh-lister +Maintainer: Software Heritage developers +Section: python +Priority: optional +Build-Depends: debhelper (>= 9), + dh-python (>= 3), + python3-all, + python3-arrow, + python3-bs4, + python3-dateutil, + python3-debian, + python3-iso8601, + python3-launchpadlib, + python3-testresources, + python3-pytest, + python3-pytest-mock, + python3-requests, + python3-requests-mock, + python3-setuptools, + python3-sqlalchemy (>= 1.0), + python3-swh.core (>= 0.9), + python3-swh.core.db.pytestplugin (>= 0.9), + python3-swh.scheduler (>= 0.0.31~), + python3-swh.storage (>= 0.0.103~), + python3-tenacity, + python3-testing.postgresql, + python3-setuptools-scm, + python3-xmltodict +Standards-Version: 3.9.6 +Homepage: https://forge.softwareheritage.org/source/swh-lister/ + +Package: python3-swh.lister +Architecture: all +Depends: python3-swh.core (>= 0.9), + python3-swh.scheduler (>= 0.0.31~), + python3-swh.storage (>= 0.0.103~), + ${misc:Depends}, + ${python3:Depends} +Breaks: python3-swh.lister.github +Replaces: python3-swh.lister.github +Description: Software Heritage Listers (bitbucket, git(lab|hub), pypi, etc...) diff --git a/debian/copyright b/debian/copyright new file mode 100644 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,22 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ + +Files: * +Copyright: 2016 The Software Heritage developers +License: GPL-3+ + +License: GPL-3+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + . + On Debian systems, the complete text of the GNU General Public + License version 3 can be found in `/usr/share/common-licenses/GPL-3'. diff --git a/debian/gbp.conf b/debian/gbp.conf new file mode 100644 --- /dev/null +++ b/debian/gbp.conf @@ -0,0 +1,6 @@ +[DEFAULT] +upstream-branch=debian/upstream +upstream-tag=debian/upstream/%(version)s +upstream-vcs-tag=v%(version)s +debian-branch=debian/unstable-swh +pristine-tar=True diff --git a/debian/pybuild.testfiles b/debian/pybuild.testfiles new file mode 100644 --- /dev/null +++ b/debian/pybuild.testfiles @@ -0,0 +1 @@ +swh.lister.egg-info diff --git a/debian/rules b/debian/rules new file mode 100755 --- /dev/null +++ b/debian/rules @@ -0,0 +1,11 @@ +#!/usr/bin/make -f + +export PYBUILD_NAME=swh.lister +export PYBUILD_TEST_ARGS=-v -m "not db and not fs" + +%: + dh $@ --with python3 --buildsystem=pybuild + +override_dh_install: + dh_install + rm -v $(CURDIR)/debian/python3-*/usr/lib/python*/dist-packages/swh/__init__.py diff --git a/debian/source/format b/debian/source/format new file mode 100644 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/mypy.ini b/mypy.ini --- a/mypy.ini +++ b/mypy.ini @@ -20,6 +20,12 @@ [mypy-iso8601.*] ignore_missing_imports = True +[mypy-launchpadlib.*] +ignore_missing_imports = True + +[mypy-lazr.*] +ignore_missing_imports = True + [mypy-pkg_resources.*] ignore_missing_imports = True diff --git a/swh/lister/launchpad/__init__.py b/swh/lister/launchpad/__init__.py --- a/swh/lister/launchpad/__init__.py +++ b/swh/lister/launchpad/__init__.py @@ -1,14 +1,12 @@ -# Copyright (C) 2020 the Software Heritage developers +# Copyright (C) 2020-2021 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information def register(): from .lister import LaunchpadLister - from .models import LaunchpadModel return { - "models": [LaunchpadModel], "lister": LaunchpadLister, "task_modules": ["%s.tasks" % __name__], } diff --git a/swh/lister/launchpad/lister.py b/swh/lister/launchpad/lister.py --- a/swh/lister/launchpad/lister.py +++ b/swh/lister/launchpad/lister.py @@ -1,128 +1,127 @@ -# Copyright (C) 2017-2020 The Software Heritage developers +# Copyright (C) 2020-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from datetime import datetime, timedelta -from itertools import count -from typing import Any, Dict, List, Optional, Tuple, Union +from dataclasses import asdict, dataclass +from datetime import datetime +import logging +from typing import Any, Dict, Iterator, Optional -from launchpadlib.launchpad import Launchpad # type: ignore -from lazr.restfulclient.resource import Collection, Entry # type: ignore -from sqlalchemy import func +import iso8601 +from launchpadlib.launchpad import Launchpad +from lazr.restfulclient.resource import Collection -from swh.lister.core.lister_base import ListerBase +from swh.scheduler.interface import SchedulerInterface +from swh.scheduler.model import ListedOrigin -from .models import LaunchpadModel +from ..pattern import CredentialsType, Lister +logger = logging.getLogger(__name__) + +LaunchpadPageType = Iterator[Collection] + + +@dataclass +class LaunchpadListerState: + """State of Launchpad lister""" + + date_last_modified: Optional[datetime] = None + """modification date of last updated repository since last listing""" + + +class LaunchpadLister(Lister[LaunchpadListerState, LaunchpadPageType]): + """ + List git repositories from Launchpad. + + Args: + scheduler: instance of SchedulerInterface + incremental: defines if incremental listing should be used, in that case + only modified or new repositories since last incremental listing operation + will be returned + """ -class LaunchpadLister(ListerBase): - MODEL = LaunchpadModel LISTER_NAME = "launchpad" - instance = "launchpad" - flush_packet_db = 20 - def __init__(self, override_config=None): - super().__init__(override_config=override_config) + def __init__( + self, + scheduler: SchedulerInterface, + incremental: bool = False, + credentials: CredentialsType = None, + ): + super().__init__( + scheduler=scheduler, + url="https://launchpad.net/", + instance="launchpad", + credentials=credentials, + ) + self.incremental = incremental + self.date_last_modified = None + + def state_from_dict(self, d: Dict[str, Any]) -> LaunchpadListerState: + date_last_modified = d.get("date_last_modified") + if date_last_modified is not None: + d["date_last_modified"] = iso8601.parse_date(date_last_modified) + return LaunchpadListerState(**d) + + def state_to_dict(self, state: LaunchpadListerState) -> Dict[str, Any]: + d = asdict(state) + date_last_modified = d.get("date_last_modified") + if date_last_modified is not None: + d["date_last_modified"] = date_last_modified.isoformat() + return d + + def get_pages(self) -> Iterator[LaunchpadPageType]: + """ + Yields an iterator on all git repositories hosted on Launchpad sorted + by last modification date in ascending order. + """ launchpad = Launchpad.login_anonymously( "softwareheritage", "production", version="devel" ) - self.get_repos = launchpad.git_repositories.getRepositories - - def get_model_from_repo(self, repo: Entry) -> Dict[str, Union[str, datetime]]: - return { - "uid": repo.unique_name, - "name": repo.name, - "full_name": repo.name, - "origin_url": repo.git_https_url, - "html_url": repo.web_link, - "origin_type": "git", - "date_last_modified": repo.date_last_modified, - } - - def lib_response_simplified( - self, response: Collection - ) -> List[Dict[str, Union[str, datetime]]]: - return [ - self.get_model_from_repo(repo) for repo in response[: len(response.entries)] - ] - - def db_last_threshold(self) -> Optional[datetime]: - t = self.db_session.query(func.max(self.MODEL.date_last_modified)).first() - if t: - return t[0] - else: - return None - - def ingest_data_lp( - self, identifier: Optional[datetime], checks: bool = False - ) -> Tuple[Collection, dict]: - """ The core data fetch sequence. Request launchpadlib endpoint. Simplify and - filter response list of repositories. Inject repo information into - local db. Queue loader tasks for linked repositories. - - Args: - identifier: Resource identifier. - checks: Additional checks required - """ - response = self.get_repos( - order_by="most neglected first", modified_since_date=identifier + date_last_modified = None + if self.incremental: + date_last_modified = self.state.date_last_modified + get_repos = launchpad.git_repositories.getRepositories + yield get_repos( + order_by="most neglected first", modified_since_date=date_last_modified ) - models_list = self.lib_response_simplified(response) - models_list = self.filter_before_inject(models_list) - if checks: - models_list = self.do_additional_checks(models_list) - if not models_list: - return response, {} - # inject into local db - injected = self.inject_repo_data_into_db(models_list) - # queue workers - self.schedule_missing_tasks(models_list, injected) - return response, injected - - def run(self, max_bound: Optional[datetime] = None) -> Dict[str, Any]: - """ Main entry function. Sequentially fetches repository data - from the service according to the basic outline in the class - docstring, continually fetching sublists until either there - is no next index reference given or the given next index is greater - than the desired max_bound. - - Args: - max_bound : optional date to start at - Returns: - Dict containing listing status + + def get_origins_from_page(self, page: LaunchpadPageType) -> Iterator[ListedOrigin]: """ - status = "uneventful" + Iterate on all git repositories and yield ListedOrigin instances. + """ + assert self.lister_obj.id is not None + + for repo in page: + + origin_url = repo.git_https_url - def ingest_git_repos(): - threshold = max_bound - for i in count(1): - response, injected_repos = self.ingest_data_lp(threshold) - if not response and not injected_repos: - return + # filter out origins with invalid URL + if not origin_url.startswith("https://"): + continue - # batch is empty - if len(response.entries) == 0: - return + last_update = repo.date_last_modified - first: datetime = response[0].date_last_modified - last: datetime = response[len(response.entries) - 1].date_last_modified + self.date_last_modified = last_update - next_date = last - timedelta(seconds=15) + logger.debug("Found origin %s last updated on %s", origin_url, last_update) - if next_date <= first: - delta = last - first - next_date = last - delta / 2 + yield ListedOrigin( + lister_id=self.lister_obj.id, + visit_type="git", + url=origin_url, + last_update=last_update, + ) - threshold = next_date - yield i + def finalize(self) -> None: + if self.date_last_modified is None: + return - for i in ingest_git_repos(): - if (i % self.flush_packet_db) == 0: - self.db_session.commit() - self.db_session = self.mk_session() - status = "eventful" + if self.incremental and ( + self.state.date_last_modified is None + or self.date_last_modified > self.state.date_last_modified + ): + self.state.date_last_modified = self.date_last_modified - self.db_session.commit() - self.db_session = self.mk_session() - return {"status": status} + self.updated = True diff --git a/swh/lister/launchpad/models.py b/swh/lister/launchpad/models.py deleted file mode 100644 --- a/swh/lister/launchpad/models.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (C) 2017-2020 the Software Heritage developers -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from sqlalchemy import Column, Date, String - -from swh.lister.core.models import ModelBase - - -class LaunchpadModel(ModelBase): - """a Launchpad repository""" - - __tablename__ = "launchpad_repo" - - uid = Column(String, primary_key=True) - date_last_modified = Column(Date, index=True) diff --git a/swh/lister/launchpad/tasks.py b/swh/lister/launchpad/tasks.py --- a/swh/lister/launchpad/tasks.py +++ b/swh/lister/launchpad/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2020 The Software Heritage developers +# Copyright (C) 2020-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -8,27 +8,20 @@ from .lister import LaunchpadLister +@shared_task(name=__name__ + ".ping") +def ping(): + return "OK" + + +@shared_task(name=__name__ + ".FullLaunchpadLister") +def list_launchpad_full(**lister_args): + """Full listing of git repositories hosted on Launchpad""" + lister = LaunchpadLister.from_configfile(**lister_args) + return lister.run().dict() + + @shared_task(name=__name__ + ".IncrementalLaunchpadLister") -def list_launchpad_incremental(threshold, **lister_args): - """Incremental update - """ - lister = LaunchpadLister(**lister_args) - return lister.run(max_bound=threshold) - - -@shared_task(name=__name__ + ".FullLaunchpadLister", bind=True) -def list_launchpad_full(self, **lister_args): - """Full update of Launchpad - """ - self.log.debug("%s OK, spawned full task" % (self.name)) - return list_launchpad_incremental(threshold=None, **lister_args) - - -@shared_task(name=__name__ + ".NewLaunchpadLister", bind=True) -def list_launchpad_new(self, **lister_args): - """Update new entries of Launchpad - """ - lister = LaunchpadLister(**lister_args) - threshold = lister.db_last_threshold() - self.log.debug("%s OK, spawned new task" % (self.name)) - return list_launchpad_incremental(threshold=threshold, **lister_args) +def list_launchpad_incremental(**lister_args): + """Incremental listing of git repositories hosted on Launchpad""" + lister = LaunchpadLister.from_configfile(**lister_args, incremental=True) + return lister.run().dict() diff --git a/swh/lister/launchpad/tests/data/response0.json b/swh/lister/launchpad/tests/data/launchpad_response1.json rename from swh/lister/launchpad/tests/data/response0.json rename to swh/lister/launchpad/tests/data/launchpad_response1.json diff --git a/swh/lister/launchpad/tests/data/response1.json b/swh/lister/launchpad/tests/data/launchpad_response2.json rename from swh/lister/launchpad/tests/data/response1.json rename to swh/lister/launchpad/tests/data/launchpad_response2.json diff --git a/swh/lister/launchpad/tests/data/response2.json b/swh/lister/launchpad/tests/data/response2.json deleted file mode 100644 --- a/swh/lister/launchpad/tests/data/response2.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/swh/lister/launchpad/tests/test_lister.py b/swh/lister/launchpad/tests/test_lister.py --- a/swh/lister/launchpad/tests/test_lister.py +++ b/swh/lister/launchpad/tests/test_lister.py @@ -1,30 +1,156 @@ -# Copyright (C) 2020 The Software Heritage developers +# Copyright (C) 2020-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from datetime import datetime +import json +from pathlib import Path +from typing import List -def test_launchpad_lister(lister_launchpad, datadir): - lister_launchpad.run() +import pytest - assert len(lister_launchpad.get_repos.mock_calls) == 3 +from ..lister import LaunchpadLister - r = lister_launchpad.scheduler.search_tasks(task_type="load-git") - assert len(r) == 30 - for row in r: - assert row["type"] == "load-git" - # arguments check - args = row["arguments"]["args"] - assert len(args) == 0 +class _Repo: + def __init__(self, d: dict): + for key in d.keys(): + if key == "date_last_modified": + setattr(self, key, datetime.fromisoformat(d[key])) + else: + setattr(self, key, d[key]) - # kwargs - kwargs = row["arguments"]["kwargs"] - assert set(kwargs.keys()) == {"url"} - url = kwargs["url"] - assert url.startswith("https://git.launchpad.net") +class _Collection: + entries: List[_Repo] = [] - assert row["policy"] == "recurring" - assert row["priority"] is None - assert row["retries_left"] == 3 + def __init__(self, file): + self.entries = [_Repo(r) for r in file] + + def __getitem__(self, key): + return self.entries[key] + + def __len__(self): + return len(self.entries) + + +def _launchpad_response(datadir, datafile): + return _Collection(json.loads(Path(datadir, datafile).read_text())) + + +@pytest.fixture +def launchpad_response1(datadir): + return _launchpad_response(datadir, "launchpad_response1.json") + + +@pytest.fixture +def launchpad_response2(datadir): + return _launchpad_response(datadir, "launchpad_response2.json") + + +def _mock_getRepositories(mocker, launchpad_response): + mock_launchpad = mocker.patch("swh.lister.launchpad.lister.Launchpad") + mock_getRepositories = mock_launchpad.git_repositories.getRepositories + mock_getRepositories.return_value = launchpad_response + mock_launchpad.login_anonymously.return_value = mock_launchpad + + return mock_getRepositories + + +def _check_listed_origins(scheduler_origins, launchpad_response): + for origin in launchpad_response: + + filtered_origins = [ + o for o in scheduler_origins if o.url == origin.git_https_url + ] + + assert len(filtered_origins) == 1 + + assert filtered_origins[0].last_update == origin.date_last_modified + + +def test_lister_from_configfile(swh_scheduler_config, mocker): + load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar") + load_from_envvar.return_value = { + "scheduler": {"cls": "local", **swh_scheduler_config}, + "credentials": {}, + } + lister = LaunchpadLister.from_configfile() + assert lister.scheduler is not None + assert lister.credentials is not None + + +def test_launchpad_full_lister(swh_scheduler, mocker, launchpad_response1): + mock_getRepositories = _mock_getRepositories(mocker, launchpad_response1) + lister = LaunchpadLister(scheduler=swh_scheduler) + stats = lister.run() + + assert not lister.incremental + assert lister.updated + assert stats.pages == 1 + assert stats.origins == len(launchpad_response1) + + mock_getRepositories.assert_called_once_with( + order_by="most neglected first", modified_since_date=None + ) + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results + + assert len(scheduler_origins) == len(launchpad_response1) + + _check_listed_origins(scheduler_origins, launchpad_response1) + + +def test_launchpad_incremental_lister( + swh_scheduler, mocker, launchpad_response1, launchpad_response2 +): + mock_getRepositories = _mock_getRepositories(mocker, launchpad_response1) + lister = LaunchpadLister(scheduler=swh_scheduler, incremental=True) + stats = lister.run() + + assert lister.incremental + assert lister.updated + assert stats.pages == 1 + assert stats.origins == len(launchpad_response1) + + mock_getRepositories.assert_called_once_with( + order_by="most neglected first", modified_since_date=None + ) + + lister_state = lister.get_state_from_scheduler() + assert lister_state.date_last_modified == launchpad_response1[-1].date_last_modified + + mock_getRepositories = _mock_getRepositories(mocker, launchpad_response2) + lister = LaunchpadLister(scheduler=swh_scheduler, incremental=True) + stats = lister.run() + + assert lister.incremental + assert lister.updated + assert stats.pages == 1 + assert stats.origins == len(launchpad_response2) + + mock_getRepositories.assert_called_once_with( + order_by="most neglected first", + modified_since_date=lister_state.date_last_modified, + ) + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results + + assert len(scheduler_origins) == len(launchpad_response1) + len(launchpad_response2) + + _check_listed_origins(scheduler_origins, launchpad_response1) + _check_listed_origins(scheduler_origins, launchpad_response2) + + +def test_launchpad_lister_invalid_url_filtering( + swh_scheduler, mocker, +): + invalid_origin = [_Repo({"git_https_url": "tag:launchpad.net:2008:redacted",})] + _mock_getRepositories(mocker, invalid_origin) + lister = LaunchpadLister(scheduler=swh_scheduler) + stats = lister.run() + + assert not lister.updated + assert stats.pages == 1 + assert stats.origins == 0 diff --git a/swh/lister/launchpad/tests/test_tasks.py b/swh/lister/launchpad/tests/test_tasks.py --- a/swh/lister/launchpad/tests/test_tasks.py +++ b/swh/lister/launchpad/tests/test_tasks.py @@ -1,42 +1,54 @@ -# Copyright (C) 2020 The Software Heritage developers +# Copyright (C) 2020-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from unittest.mock import patch +from swh.lister.pattern import ListerStats -@patch("swh.lister.launchpad.tasks.LaunchpadLister") -def test_new(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): - # setup the mocked LaunchpadLister - lister.return_value = lister - lister.run.return_value = None +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.launchpad.tasks.ping") + assert res + res.wait() + assert res.successful() + assert res.result == "OK" + + +def test_launchpad_full_listing_task( + swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker +): + lister = mocker.patch("swh.lister.launchpad.tasks.LaunchpadLister") + lister.from_configfile.return_value = lister + stats = ListerStats(pages=1, origins=28000) + lister.run.return_value = stats res = swh_scheduler_celery_app.send_task( - "swh.lister.launchpad.tasks.NewLaunchpadLister" + "swh.lister.launchpad.tasks.FullLaunchpadLister" ) assert res res.wait() assert res.successful() + assert res.result == stats.dict() - assert lister.call_count == 2 - lister.db_last_threshold.assert_called_once() - lister.run.assert_called_once() + lister.from_configfile.assert_called_once_with() + lister.run.assert_called_once_with() -@patch("swh.lister.launchpad.tasks.LaunchpadLister") -def test_full(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): - # setup the mocked LaunchpadLister - lister.return_value = lister - lister.run.return_value = None +def test_launchpad_incremental_listing_task( + swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker +): + lister = mocker.patch("swh.lister.launchpad.tasks.LaunchpadLister") + lister.from_configfile.return_value = lister + stats = ListerStats(pages=1, origins=200) + lister.run.return_value = stats res = swh_scheduler_celery_app.send_task( - "swh.lister.launchpad.tasks.FullLaunchpadLister" + "swh.lister.launchpad.tasks.IncrementalLaunchpadLister" ) assert res res.wait() assert res.successful() + assert res.result == stats.dict() - lister.assert_called_once() - lister.db_last_threshold.assert_not_called() - lister.run.assert_called_once_with(max_bound=None) + lister.from_configfile.assert_called_once_with(incremental=True) + lister.run.assert_called_once_with() diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -36,8 +36,7 @@ db_url = init_db().url() # Drop launchpad lister from the lister to check, its test setup is more involved # than the other listers and it's not currently done here - supported_listers = set(SUPPORTED_LISTERS) - {"launchpad"} - for lister_name in supported_listers: + for lister_name in SUPPORTED_LISTERS: lst = get_lister( lister_name, db_url,