No OneTemporary
Actions

Size

84 KB

Subscribers

None

View Options

	diff --git a/PKG-INFO b/PKG-INFO
	index 186901f..d724363 100644
	--- a/PKG-INFO
	+++ b/PKG-INFO
	@@ -1,30 +1,30 @@
	Metadata-Version: 2.1
	Name: swh.loader.core
	-Version: 0.0.84
	+Version: 0.0.85
	Summary: Software Heritage Base Loader
	Home-page: https://forge.softwareheritage.org/diffusion/DLDBASE
	Author: Software Heritage developers
	Author-email: swh-devel@inria.fr
	License: UNKNOWN
	Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
	Project-URL: Funding, https://www.softwareheritage.org/donate
	Project-URL: Source, https://forge.softwareheritage.org/source/swh-loader-core
	Description: SWH-loader-core
	===============

	The Software Heritage Core Loader is a low-level loading utilities and
	helpers used by other loaders.

	The main entry points are classes:
	- :class:`swh.loader.core.loader.BaseLoader` for loaders (e.g. svn)
	- :class:`swh.loader.core.loader.DVCSLoader` for DVCS loaders (e.g. hg, git, ...)
	- :class:`swh.loader.package.loader.PackageLoader` for Package loaders (e.g. PyPI, Npm, ...)

	Platform: UNKNOWN
	Classifier: Programming Language :: Python :: 3
	Classifier: Intended Audience :: Developers
	Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
	Classifier: Operating System :: OS Independent
	Classifier: Development Status :: 5 - Production/Stable
	Description-Content-Type: text/markdown
	Provides-Extra: testing
	diff --git a/conftest.py b/conftest.py
	index b669dfa..23c9f57 100644
	--- a/conftest.py
	+++ b/conftest.py
	@@ -1,76 +1,76 @@
	# Copyright (C) 2019-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import os
	import pytest
	import yaml

	from typing import Any, Dict

	from swh.storage.tests.conftest import * # noqa
	from swh.scheduler.tests.conftest import * # noqa


	@pytest.fixture
	def swh_loader_config(swh_storage_postgresql) -> Dict[str, Any]:
	return {
	'storage': {
	'cls': 'pipeline',
	'steps': [
	{'cls': 'retry'},
	{'cls': 'filter'},
	{'cls': 'buffer'},
	{
	'cls': 'local',
	'args': {
	'db': swh_storage_postgresql.dsn,
	'objstorage': {
	'cls': 'memory',
	'args': {}
	},
	}
	}
	]
	},
	'deposit': {
	'url': 'https://deposit.softwareheritage.org/1/private',
	'auth': {
	'username': 'user',
	'password': 'pass',
	}
	},
	}


	@pytest.fixture
	def swh_config(swh_loader_config, monkeypatch, tmp_path):
	conffile = os.path.join(str(tmp_path), 'loader.yml')
	with open(conffile, 'w') as f:
	f.write(yaml.dump(swh_loader_config))
	monkeypatch.setenv('SWH_CONFIG_FILENAME', conffile)
	return conffile


	@pytest.fixture(autouse=True, scope='session')
	def swh_proxy():
	"""Automatically inject this fixture in all tests to ensure no outside
	connection takes place.

	"""
	os.environ['http_proxy'] = 'http://localhost:999'
	os.environ['https_proxy'] = 'http://localhost:999'


	@pytest.fixture(scope='session') # type: ignore # expected redefinition
	def celery_includes():
	return [
	'swh.loader.package.archive.tasks',
	'swh.loader.package.cran.tasks',
	'swh.loader.package.debian.tasks',
	'swh.loader.package.deposit.tasks',
	'swh.loader.package.npm.tasks',
	'swh.loader.package.pypi.tasks',
	- 'swh.loader.package.functional.tasks',
	+ 'swh.loader.package.nixguix.tasks',
	]
	diff --git a/debian/changelog b/debian/changelog
	index 763f9da..78b7d74 100644
	--- a/debian/changelog
	+++ b/debian/changelog
	@@ -1,792 +1,797 @@
	-swh-loader-core (0.0.84-1~swh1~bpo10+1) buster-swh; urgency=medium
	+swh-loader-core (0.0.85-1~swh1) unstable-swh; urgency=medium

	- * Rebuild for buster-swh
	+ * New upstream release 0.0.85 - (tagged by Antoine R. Dumont
	+ (@ardumont) <antoine.romain.dumont@gmail.com> on 2020-03-26 15:36:58
	+ +0100)
	+ * Upstream changes: - v0.0.85 - core.loader: Allow core loader
	+ to update origin_visit in one call - Rename the functional
	+ loader to nixguix loader

	- -- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 24 Mar 2020 10:34:59 +0000
	+ -- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 26 Mar 2020 14:43:17 +0000

	swh-loader-core (0.0.84-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.84 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-03-24 11:29:49
	+0100)
	* Upstream changes: - v0.0.84 - test: Use storage endpoint to
	check latest origin visit status - package.loader: Fix status
	visit to 'partial' - package.loader: add a test to reproduce
	EOFError error

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 24 Mar 2020 10:32:55 +0000

	swh-loader-core (0.0.83-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.83 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-03-23 15:16:14
	+0100)
	* Upstream changes: - v0.0.83 - Make the swh.loader.package
	exception handling more granular - package.loader: Reference a
	snapshot on partial visit - package.loader: Extract a
	_load_snapshot method - functional: create a branch named
	evaluation pointing to the evaluation commit - package.loader:
	add extra_branches method

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Mon, 23 Mar 2020 14:19:43 +0000

	swh-loader-core (0.0.82-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.82 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-03-18 11:55:48
	+0100)
	* Upstream changes: - v0.0.82 - functional.loader: Add loader
	- package.loader: ignore non tarball source

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Wed, 18 Mar 2020 10:59:38 +0000

	swh-loader-core (0.0.81-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.81 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-03-16 13:14:33
	+0100)
	* Upstream changes: - v0.0.81 - Migrate to latest
	storage.origin_visit_add api change - Move Person parsing to swh-
	model.

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Mon, 16 Mar 2020 12:17:43 +0000

	swh-loader-core (0.0.80-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.80 - (tagged by Valentin Lorentz
	<vlorentz@softwareheritage.org> on 2020-02-28 17:05:14 +0100)
	* Upstream changes: - v0.0.80 - * use swh-model objects
	instead of dicts.

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 28 Feb 2020 16:10:06 +0000

	swh-loader-core (0.0.79-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.79 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-02-25 11:40:05
	+0100)
	* Upstream changes: - v0.0.79 - Move revision loading logic to
	its own function. - Use swh-storage validation proxy earlier in
	the pipeline. - Use swh-storage validation proxy. - Add
	missing __init__.py and fix tests.

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 25 Feb 2020 10:48:07 +0000

	swh-loader-core (0.0.78-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.78 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-02-06 15:28:11
	+0100)
	* Upstream changes: - v0.0.78 - tests: Use new get_storage
	signature - loader.core.converters: Prefer the with open pattern
	to read file - test_converters: Add coverage on prepare_contents
	method - test_converters: Migrate to pytest -
	loader.core/package: Call storage's (skipped_)content_add endpoints

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 06 Feb 2020 15:09:05 +0000

	swh-loader-core (0.0.77-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.77 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-01-30 10:32:08
	+0100)
	* Upstream changes: - v0.0.77 - loader.npm: If no upload time
	provided, use artifact's mtime if provided - loader.npm: Fail
	ingestion if at least 1 artifact has no upload time

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 30 Jan 2020 09:37:58 +0000

	swh-loader-core (0.0.76-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.76 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-01-28 13:07:30
	+0100)
	* Upstream changes: - v0.0.76 - npm.loader: Skip artifacts
	with no intrinsic metadata - pypi.loader: Skip artifacts with no
	intrinsic metadata - package.loader: Fix edge case when some
	listing returns no content - core.loader: Drop retro-
	compatibility class names - loader.tests: Add filter and buffer
	proxy storage - docs: Fix sphinx warnings - README: Update
	class names

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 28 Jan 2020 12:11:07 +0000

	swh-loader-core (0.0.75-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.75 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-01-16 14:14:29
	+0100)
	* Upstream changes: - v0.0.75 - cran.loader: Align cran loader
	with other package loaders

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 16 Jan 2020 13:17:30 +0000

	swh-loader-core (0.0.74-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.74 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-01-15 15:30:13
	+0100)
	* Upstream changes: - v0.0.74 - Drop no longer used retrying
	dependency - core.loader: Clean up indirection and retry
	behavior - tests: Use retry proxy storage in loaders -
	core.loader: Drop dead code - cran.loader: Fix parsing
	description file error

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Wed, 15 Jan 2020 14:33:57 +0000

	swh-loader-core (0.0.73-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.73 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-01-09 10:00:21
	+0100)
	* Upstream changes: - v0.0.73 - package.cran: Name CRAN task
	appropriately

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 09 Jan 2020 09:05:07 +0000

	swh-loader-core (0.0.72-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.72 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2020-01-06 16:37:58
	+0100)
	* Upstream changes: - v0.0.72 - package.loader: Fail fast when
	unable to create origin/origin_visit - cran.loader: Add
	implementation

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Mon, 06 Jan 2020 15:50:08 +0000

	swh-loader-core (0.0.71-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.71 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-20 14:22:31
	+0100)
	* Upstream changes: - v0.0.71 - package.utils: Drop unneeded
	hashes from download computation

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 20 Dec 2019 13:26:09 +0000

	swh-loader-core (0.0.70-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.70 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-20 11:32:09
	+0100)
	* Upstream changes: - v0.0.70 - debian.loader: Improve and fix
	revision resolution's corner cases

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 20 Dec 2019 10:39:34 +0000

	swh-loader-core (0.0.69-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.69 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-12 16:21:59
	+0100)
	* Upstream changes: - v0.0.69 - loader.core: Fix correctly
	loader initialization

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 12 Dec 2019 15:26:13 +0000

	swh-loader-core (0.0.68-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.68 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-12 15:45:21
	+0100)
	* Upstream changes: - v0.0.68 - loader.core: Fix
	initialization issue in dvcs loaders

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 12 Dec 2019 14:49:12 +0000

	swh-loader-core (0.0.67-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.67 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-12 14:02:47
	+0100)
	* Upstream changes: - v0.0.67 - loader.core: Type methods -
	loader.core: Transform data input into list - loader.core: Add
	missing conversion step on content

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 12 Dec 2019 13:07:47 +0000

	swh-loader-core (0.0.66-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.66 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-12 12:01:14
	+0100)
	* Upstream changes: - v0.0.66 - Drop deprecated behavior

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 12 Dec 2019 11:05:17 +0000

	swh-loader-core (0.0.65-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.65 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-12 11:42:46
	+0100)
	* Upstream changes: - v0.0.65 - loader.cli: Improve current
	implementation - tasks: Enforce kwargs use in task message

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 12 Dec 2019 10:51:02 +0000

	swh-loader-core (0.0.64-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.64 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-10 09:49:06
	+0100)
	* Upstream changes: - v0.0.64 - requirements-test: Add missing
	test dependency - tests: Refactor using pytest-mock's mocker
	fixture - loader.cli: Add tests around cli - package.npm:
	Align loader instantiation - loader.cli: Reference new loader
	cli

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 10 Dec 2019 08:56:02 +0000

	swh-loader-core (0.0.63-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.63 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-05 16:01:49
	+0100)
	* Upstream changes: - v0.0.63 - Add missing inclusion
	instruction

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 05 Dec 2019 15:05:39 +0000

	swh-loader-core (0.0.62-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.62 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-12-05 15:46:46
	+0100)
	* Upstream changes: - v0.0.62 - Move package loaders to their
	own namespace

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 05 Dec 2019 14:50:19 +0000

	swh-loader-core (0.0.61-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.61 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-28 17:25:49
	+0100)
	* Upstream changes: - v0.0.61 - pypi: metadata -> revision:
	Deal with previous metadata format - npm: metadata -> revision:
	Deal with previous metadata format

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 28 Nov 2019 16:29:47 +0000

	swh-loader-core (0.0.60-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.60 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-26 12:09:28
	+0100)
	* Upstream changes: - v0.0.60 - package.deposit: Fix revision-
	get inconsistency - package.deposit: Provide parents in any case
	- package.deposit: Fix url computation issue - utils: Work
	around header issue during download

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 26 Nov 2019 11:18:41 +0000

	swh-loader-core (0.0.59-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.59 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-22 18:11:33
	+0100)
	* Upstream changes: - v0.0.59 - npm: Explicitely retrieve the
	revision date from extrinsic metadata

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 22 Nov 2019 17:15:34 +0000

	swh-loader-core (0.0.58-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.58 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-22 12:08:10
	+0100)
	* Upstream changes: - v0.0.58 - package.pypi: Filter out non-
	sdist package type

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 22 Nov 2019 11:11:56 +0000

	swh-loader-core (0.0.57-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.57 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-22 11:26:11
	+0100)
	* Upstream changes: - v0.0.57 - package.pypi: Fix project url
	computation edge case - Use pkg_resources to get the package
	version instead of vcversioner

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 22 Nov 2019 10:31:11 +0000

	swh-loader-core (0.0.56-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.56 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-21 16:12:46
	+0100)
	* Upstream changes: - v0.0.56 - package.tasks: Rename
	appropriately load_deb_package task type name - Fix typos
	reported by codespell - Add a pre-commit config file

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 21 Nov 2019 15:16:23 +0000

	swh-loader-core (0.0.55-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.55 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-21 13:51:03
	+0100)
	* Upstream changes: - v0.0.55 - package.tasks: Rename
	load_archive into load_archive_files - Migrate tox.ini to extras
	= xxx instead of deps = .[testing] - Merge tox test environments

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 21 Nov 2019 12:56:07 +0000

	swh-loader-core (0.0.54-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.54 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-21 11:29:20
	+0100)
	* Upstream changes: - v0.0.54 - loader.package.deposit: Drop
	swh.deposit.client requirement - Include all requirements in
	MANIFEST.in

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 21 Nov 2019 10:32:23 +0000

	swh-loader-core (0.0.53-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.53 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-20 14:26:36
	+0100)
	* Upstream changes: - v0.0.53 - loader.package.tasks: Document
	tasks - Define correctly the setup.py's entry_points

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Wed, 20 Nov 2019 13:30:10 +0000

	swh-loader-core (0.0.52-1~swh3) unstable-swh; urgency=medium

	* Update dh-python version constraint

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 20 Nov 2019 12:03:00 +0100

	swh-loader-core (0.0.52-1~swh2) unstable-swh; urgency=medium

	* Add egg-info to pybuild.testfiles.

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 20 Nov 2019 11:42:42 +0100

	swh-loader-core (0.0.52-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.52 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-19 15:15:40
	+0100)
	* Upstream changes: - v0.0.52 - Ensure BufferedLoader and
	UnbufferedLoader do flush their storage - loader.package:
	Register loader package tasks - package.tasks: Rename debian
	task to load_deb

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 19 Nov 2019 14:18:41 +0000

	swh-loader-core (0.0.51-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.51 - (tagged by David Douard
	<david.douard@sdfa3.org> on 2019-11-18 17:05:17 +0100)
	* Upstream changes: - v0.0.51

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Mon, 18 Nov 2019 16:09:44 +0000

	swh-loader-core (0.0.50-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.50 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-13 15:56:55
	+0100)
	* Upstream changes: - v0.0.50 - package.loader: Check
	snapshot_id is set as returned value - package.loader: Ensure
	the origin visit type is set appropriately - package.loader: Fix
	serialization issue - package.debian: Align origin_visit type to
	'deb' as in production

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Wed, 13 Nov 2019 15:04:37 +0000

	swh-loader-core (0.0.49-1~swh2) unstable-swh; urgency=medium

	* Update dependencies

	-- Antoine R. Dumont <antoine.romain.dumont@gmail.com> Fri, 08 Nov 2019 14:07:20 +0100

	swh-loader-core (0.0.49-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.49 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-11-08 13:21:56
	+0100)
	* Upstream changes: - v0.0.49 - New package loader
	implementations: archive, pypi, npm, deposit, debian

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 08 Nov 2019 12:29:47 +0000

	swh-loader-core (0.0.48-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.48 - (tagged by Stefano Zacchiroli
	<zack@upsilon.cc> on 2019-10-01 16:49:39 +0200)
	* Upstream changes: - v0.0.48 - * typing: minimal changes to
	make a no-op mypy run pass

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 01 Oct 2019 14:52:59 +0000

	swh-loader-core (0.0.47-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.47 - (tagged by Antoine Lambert
	<antoine.lambert@inria.fr> on 2019-10-01 11:32:50 +0200)
	* Upstream changes: - version 0.0.47: Workaround HashCollision
	errors

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 01 Oct 2019 09:35:38 +0000

	swh-loader-core (0.0.46-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.46 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-09-06 18:30:42
	+0200)
	* Upstream changes: - v0.0.46 - pytest.ini: Remove warnings
	about our custom markers - pep8: Fix log.warning calls -
	core/loader: Fix get_save_data_path implementation - Fix
	validation errors in test.

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 06 Sep 2019 16:33:13 +0000

	swh-loader-core (0.0.45-1~swh2) unstable-swh; urgency=medium

	* Fix missing build dependency

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Tue, 03 Sep 2019 14:12:13 +0200

	swh-loader-core (0.0.45-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.45 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-09-03 10:38:36
	+0200)
	* Upstream changes: - v0.0.45 - loader: Provide visit type
	when calling origin_visit_add - loader: Drop keys 'perms' and
	'path' from content before sending to the - storage -
	swh.loader.package: Implement GNU loader - docs: add code of
	conduct document

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Tue, 03 Sep 2019 08:41:49 +0000

	swh-loader-core (0.0.44-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.44 - (tagged by Valentin Lorentz
	<vlorentz@softwareheritage.org> on 2019-06-25 12:18:27 +0200)
	* Upstream changes: - Drop use of deprecated methods
	fetch_history_*

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Wed, 26 Jun 2019 09:40:59 +0000

	swh-loader-core (0.0.43-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.43 - (tagged by Valentin Lorentz
	<vlorentz@softwareheritage.org> on 2019-06-18 16:21:58 +0200)
	* Upstream changes: - Use origin urls instead of origin ids.

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Wed, 19 Jun 2019 09:33:53 +0000

	swh-loader-core (0.0.42-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.42 - (tagged by David Douard
	<david.douard@sdfa3.org> on 2019-05-20 11:28:49 +0200)
	* Upstream changes: - v0.0.42 - update/fix requirements

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Mon, 20 May 2019 09:33:47 +0000

	swh-loader-core (0.0.41-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.41 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-04-11 11:46:00
	+0200)
	* Upstream changes: - v0.0.41 - core.loader: Migrate to latest
	snapshot_add, origin_visit_update api - core.loader: Count only
	the effectively new objects ingested - test_utils: Add coverage
	on utils module

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Thu, 11 Apr 2019 09:52:55 +0000

	swh-loader-core (0.0.40-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.40 - (tagged by Antoine Lambert
	<antoine.lambert@inria.fr> on 2019-03-29 10:57:14 +0100)
	* Upstream changes: - version 0.0.40

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Fri, 29 Mar 2019 10:02:37 +0000

	swh-loader-core (0.0.39-1~swh1) unstable-swh; urgency=medium

	* New upstream release 0.0.39 - (tagged by Antoine R. Dumont
	(@ardumont) <antoine.romain.dumont@gmail.com> on 2019-01-30 11:10:39
	+0100)
	* Upstream changes: - v0.0.39

	-- Software Heritage autobuilder (on jenkins-debian1) <jenkins@jenkins-debian1.internal.softwareheritage.org> Wed, 30 Jan 2019 10:13:56 +0000

	swh-loader-core (0.0.35-1~swh1) unstable-swh; urgency=medium

	* v0.0.35
	* tests: Initialize tox.ini use
	* tests, debian/*: Migrate to pytest

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Tue, 23 Oct 2018 15:47:22 +0200

	swh-loader-core (0.0.34-1~swh1) unstable-swh; urgency=medium

	* v0.0.34
	* setup: prepare for PyPI upload
	* README.md: Simplify module description
	* core.tests: Install tests fixture for derivative loaders to use

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Tue, 09 Oct 2018 14:11:29 +0200

	swh-loader-core (0.0.33-1~swh1) unstable-swh; urgency=medium

	* v0.0.33
	* loader/utils: Add clean_dangling_folders function to ease clean up
	* loader/core: Add optional pre_cleanup for dangling files cleaning

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Fri, 09 Mar 2018 14:41:17 +0100

	swh-loader-core (0.0.32-1~swh1) unstable-swh; urgency=medium

	* v0.0.32
	* Improve origin_visit initialization step
	* Properly sandbox the prepare statement so that if it breaks, we can
	* update appropriately the visit with the correct status

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 07 Mar 2018 11:06:27 +0100

	swh-loader-core (0.0.31-1~swh1) unstable-swh; urgency=medium

	* Release swh.loader.core v0.0.31
	* Remove backwards-compatibility when sending snapshots

	-- Nicolas Dandrimont <nicolas@dandrimont.eu> Tue, 13 Feb 2018 18:52:20 +0100

	swh-loader-core (0.0.30-1~swh1) unstable-swh; urgency=medium

	* Release swh.loader.core v0.0.30
	* Update Debian metadata for snapshot-related breakage

	-- Nicolas Dandrimont <nicolas@dandrimont.eu> Tue, 06 Feb 2018 14:22:53 +0100

	swh-loader-core (0.0.29-1~swh1) unstable-swh; urgency=medium

	* Release swh.loader.core v0.0.29
	* Replace occurrences with snapshots
	* Enhance logging on error cases

	-- Nicolas Dandrimont <nicolas@dandrimont.eu> Tue, 06 Feb 2018 14:13:11 +0100

	swh-loader-core (0.0.28-1~swh1) unstable-swh; urgency=medium

	* v0.0.28
	* Add stateless loader base class
	* Remove bare exception handlers

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Tue, 19 Dec 2017 17:48:09 +0100

	swh-loader-core (0.0.27-1~swh1) unstable-swh; urgency=medium

	* v0.0.27
	* Migrate from indexer's indexer_configuration to storage's tool
	notion.

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Thu, 07 Dec 2017 10:36:23 +0100

	swh-loader-core (0.0.26-1~swh1) unstable-swh; urgency=medium

	* v0.0.26
	* Fix send_provider method

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Tue, 05 Dec 2017 15:40:57 +0100

	swh-loader-core (0.0.25-1~swh1) unstable-swh; urgency=medium

	* v0.0.25
	* swh.loader.core: Fix to retrieve the provider_id as an actual id
	* swh.loader.core: Fix log format error
	* swh.loader.core: Align log message according to conventions

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 29 Nov 2017 12:55:45 +0100

	swh-loader-core (0.0.24-1~swh1) unstable-swh; urgency=medium

	* v0.0.24
	* Added metadata injection possible from loader core

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Fri, 24 Nov 2017 11:35:40 +0100

	swh-loader-core (0.0.23-1~swh1) unstable-swh; urgency=medium

	* v0.0.23
	* loader: Fix dangling data flush

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Tue, 07 Nov 2017 16:25:20 +0100

	swh-loader-core (0.0.22-1~swh1) unstable-swh; urgency=medium

	* v0.0.22
	* core.loader: Use the global setup set in swh.core.config
	* core.loader: Properly batch object insertions for big requests

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Mon, 30 Oct 2017 18:50:00 +0100

	swh-loader-core (0.0.21-1~swh1) unstable-swh; urgency=medium

	* v0.0.21
	* swh.loader.core: Only send origin if not already sent before

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Tue, 24 Oct 2017 16:30:53 +0200

	swh-loader-core (0.0.20-1~swh1) unstable-swh; urgency=medium

	* v0.0.20
	* Permit to add 'post_load' actions in loaders

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Fri, 13 Oct 2017 14:30:37 +0200

	swh-loader-core (0.0.19-1~swh1) unstable-swh; urgency=medium

	* v0.0.19
	* Permit to add 'post_load' actions in loaders

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Fri, 13 Oct 2017 14:14:14 +0200

	swh-loader-core (0.0.18-1~swh1) unstable-swh; urgency=medium

	* Release swh.loader.core version 0.0.18
	* Update packaging runes

	-- Nicolas Dandrimont <nicolas@dandrimont.eu> Thu, 12 Oct 2017 18:07:53 +0200

	swh-loader-core (0.0.17-1~swh1) unstable-swh; urgency=medium

	* Release swh.loader.core v0.0.17
	* Allow iterating when fetching and storing data
	* Allow overriding the status of the loaded visit
	* Allow overriding the status of the load itself

	-- Nicolas Dandrimont <nicolas@dandrimont.eu> Wed, 11 Oct 2017 16:38:29 +0200

	swh-loader-core (0.0.16-1~swh1) unstable-swh; urgency=medium

	* Release swh.loader.core v0.0.16
	* Migrate from swh.model.git to swh.model.from_disk

	-- Nicolas Dandrimont <nicolas@dandrimont.eu> Fri, 06 Oct 2017 14:46:41 +0200

	swh-loader-core (0.0.15-1~swh1) unstable-swh; urgency=medium

	* v0.0.15
	* docs: Add sphinx apidoc generation skeleton
	* docs: Add a simple README.md explaining the module's goal
	* swh.loader.core.loader: Unify origin_visit add/update function call

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Fri, 29 Sep 2017 11:47:37 +0200

	swh-loader-core (0.0.14-1~swh1) unstable-swh; urgency=medium

	* v0.0.14
	* Add the blake2s256 hash computation

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Sat, 25 Mar 2017 18:20:52 +0100

	swh-loader-core (0.0.13-1~swh1) unstable-swh; urgency=medium

	* v0.0.13
	* Improve core loader's interface api

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 22 Feb 2017 13:43:54 +0100

	swh-loader-core (0.0.12-1~swh1) unstable-swh; urgency=medium

	* v0.0.12
	* Update storage configuration reading

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Thu, 15 Dec 2016 18:34:41 +0100

	swh-loader-core (0.0.11-1~swh1) unstable-swh; urgency=medium

	* v0.0.11
	* d/control: Bump dependency to latest storage
	* Fix: Objects can be injected even though global loading failed
	* Populate the counters in fetch_history
	* Open open/close fetch_history function in the core loader

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 24 Aug 2016 14:38:55 +0200

	swh-loader-core (0.0.10-1~swh1) unstable-swh; urgency=medium

	* v0.0.10
	* d/control: Update dependency

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Sat, 11 Jun 2016 02:26:50 +0200

	swh-loader-core (0.0.9-1~swh1) unstable-swh; urgency=medium

	* v0.0.9
	* Improve default task that initialize storage as well

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Fri, 10 Jun 2016 15:12:14 +0200

	swh-loader-core (0.0.8-1~swh1) unstable-swh; urgency=medium

	* v0.0.8
	* Migrate specific converter to the right module
	* Fix dangling parameter

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 08 Jun 2016 18:09:23 +0200

	swh-loader-core (0.0.7-1~swh1) unstable-swh; urgency=medium

	* v0.0.7
	* Fix on revision conversion

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 08 Jun 2016 16:19:02 +0200

	swh-loader-core (0.0.6-1~swh1) unstable-swh; urgency=medium

	* v0.0.6
	* d/control: Bump dependency on swh-model
	* d/control: Add missing description
	* Keep the abstraction for all entities
	* Align parameter definition order
	* Fix missing option in DEFAULT ones
	* Decrease verbosity
	* Fix missing origin_id assignment
	* d/rules: Add target to run tests during packaging

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 08 Jun 2016 16:00:40 +0200

	swh-loader-core (0.0.5-1~swh1) unstable-swh; urgency=medium

	* v0.0.5

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 25 May 2016 12:17:06 +0200

	swh-loader-core (0.0.4-1~swh1) unstable-swh; urgency=medium

	* v0.0.4
	* Rename package from python3-swh.loader to python3-swh.loader.core

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 25 May 2016 11:44:48 +0200

	swh-loader-core (0.0.3-1~swh1) unstable-swh; urgency=medium

	* v0.0.3
	* Improve default configuration
	* Rename package from swh-loader-vcs to swh-loader

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 25 May 2016 11:23:06 +0200

	swh-loader-core (0.0.2-1~swh1) unstable-swh; urgency=medium

	* v0.0.2
	* Fix: Flush data even when no data is sent to swh-storage

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Tue, 24 May 2016 16:41:49 +0200

	swh-loader-core (0.0.1-1~swh1) unstable-swh; urgency=medium

	* Initial release
	* v0.0.1

	-- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com> Wed, 13 Apr 2016 16:54:47 +0200
	diff --git a/setup.py b/setup.py
	index ee74525..620202a 100755
	--- a/setup.py
	+++ b/setup.py
	@@ -1,77 +1,77 @@
	#!/usr/bin/env python3
	# Copyright (C) 2015-2018 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	from setuptools import setup, find_packages

	from os import path
	from io import open

	here = path.abspath(path.dirname(__file__))

	# Get the long description from the README file
	with open(path.join(here, 'README.md'), encoding='utf-8') as f:
	long_description = f.read()


	def parse_requirements(name=None):
	if name:
	reqf = 'requirements-%s.txt' % name
	else:
	reqf = 'requirements.txt'

	requirements = []
	if not path.exists(reqf):
	return requirements

	with open(reqf) as f:
	for line in f.readlines():
	line = line.strip()
	if not line or line.startswith('#'):
	continue
	requirements.append(line)
	return requirements


	setup(
	name='swh.loader.core',
	description='Software Heritage Base Loader',
	long_description=long_description,
	long_description_content_type='text/markdown',
	author='Software Heritage developers',
	author_email='swh-devel@inria.fr',
	url='https://forge.softwareheritage.org/diffusion/DLDBASE',
	packages=find_packages(), # packages's modules
	scripts=[], # scripts to package
	install_requires=parse_requirements() + parse_requirements('swh'),
	setup_requires=['vcversioner'],
	extras_require={'testing': parse_requirements('test')},
	vcversioner={},
	include_package_data=True,
	entry_points='''
	[swh.workers]
	loader.archive=swh.loader.package.archive:register
	loader.cran=swh.loader.package.cran:register
	loader.debian=swh.loader.package.debian:register
	loader.deposit=swh.loader.package.deposit:register
	- loader.functional=swh.loader.package.functional:register
	+ loader.nixguix=swh.loader.package.nixguix:register
	loader.npm=swh.loader.package.npm:register
	loader.pypi=swh.loader.package.pypi:register
	[swh.cli.subcommands]
	loader=swh.loader.cli:loader
	''',
	classifiers=[
	"Programming Language :: Python :: 3",
	"Intended Audience :: Developers",
	"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
	"Operating System :: OS Independent",
	"Development Status :: 5 - Production/Stable",
	],
	project_urls={
	'Bug Reports': 'https://forge.softwareheritage.org/maniphest',
	'Funding': 'https://www.softwareheritage.org/donate',
	'Source': 'https://forge.softwareheritage.org/source/swh-loader-core',
	},
	)
	diff --git a/swh.loader.core.egg-info/PKG-INFO b/swh.loader.core.egg-info/PKG-INFO
	index 186901f..d724363 100644
	--- a/swh.loader.core.egg-info/PKG-INFO
	+++ b/swh.loader.core.egg-info/PKG-INFO
	@@ -1,30 +1,30 @@
	Metadata-Version: 2.1
	Name: swh.loader.core
	-Version: 0.0.84
	+Version: 0.0.85
	Summary: Software Heritage Base Loader
	Home-page: https://forge.softwareheritage.org/diffusion/DLDBASE
	Author: Software Heritage developers
	Author-email: swh-devel@inria.fr
	License: UNKNOWN
	Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
	Project-URL: Funding, https://www.softwareheritage.org/donate
	Project-URL: Source, https://forge.softwareheritage.org/source/swh-loader-core
	Description: SWH-loader-core
	===============

	The Software Heritage Core Loader is a low-level loading utilities and
	helpers used by other loaders.

	The main entry points are classes:
	- :class:`swh.loader.core.loader.BaseLoader` for loaders (e.g. svn)
	- :class:`swh.loader.core.loader.DVCSLoader` for DVCS loaders (e.g. hg, git, ...)
	- :class:`swh.loader.package.loader.PackageLoader` for Package loaders (e.g. PyPI, Npm, ...)

	Platform: UNKNOWN
	Classifier: Programming Language :: Python :: 3
	Classifier: Intended Audience :: Developers
	Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
	Classifier: Operating System :: OS Independent
	Classifier: Development Status :: 5 - Production/Stable
	Description-Content-Type: text/markdown
	Provides-Extra: testing
	diff --git a/swh.loader.core.egg-info/SOURCES.txt b/swh.loader.core.egg-info/SOURCES.txt
	index a294043..3344c52 100644
	--- a/swh.loader.core.egg-info/SOURCES.txt
	+++ b/swh.loader.core.egg-info/SOURCES.txt
	@@ -1,135 +1,135 @@
	MANIFEST.in
	Makefile
	README.md
	conftest.py
	requirements-swh.txt
	requirements-test.txt
	requirements.txt
	setup.py
	version.txt
	swh/__init__.py
	swh.loader.core.egg-info/PKG-INFO
	swh.loader.core.egg-info/SOURCES.txt
	swh.loader.core.egg-info/dependency_links.txt
	swh.loader.core.egg-info/entry_points.txt
	swh.loader.core.egg-info/requires.txt
	swh.loader.core.egg-info/top_level.txt
	swh/loader/__init__.py
	swh/loader/cli.py
	swh/loader/core/__init__.py
	swh/loader/core/converters.py
	swh/loader/core/loader.py
	swh/loader/core/py.typed
	swh/loader/core/utils.py
	swh/loader/core/tests/__init__.py
	swh/loader/core/tests/test_converters.py
	swh/loader/core/tests/test_loader.py
	swh/loader/core/tests/test_tests.py
	swh/loader/core/tests/test_utils.py
	swh/loader/package/__init__.py
	swh/loader/package/loader.py
	swh/loader/package/utils.py
	swh/loader/package/archive/__init__.py
	swh/loader/package/archive/loader.py
	swh/loader/package/archive/tasks.py
	swh/loader/package/archive/tests/__init__.py
	swh/loader/package/archive/tests/test_archive.py
	swh/loader/package/archive/tests/test_tasks.py
	swh/loader/package/archive/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.1.0.tar.gz
	swh/loader/package/archive/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.2.0.tar.gz
	swh/loader/package/cran/__init__.py
	swh/loader/package/cran/loader.py
	swh/loader/package/cran/tasks.py
	swh/loader/package/cran/tests/__init__.py
	swh/loader/package/cran/tests/test_cran.py
	swh/loader/package/cran/tests/test_tasks.py
	swh/loader/package/cran/tests/data/description/KnownBR
	swh/loader/package/cran/tests/data/description/acepack
	swh/loader/package/cran/tests/data/https_cran.r-project.org/src_contrib_1.4.0_Recommended_KernSmooth_2.22-6.tar.gz
	swh/loader/package/debian/__init__.py
	swh/loader/package/debian/loader.py
	swh/loader/package/debian/tasks.py
	swh/loader/package/debian/tests/__init__.py
	swh/loader/package/debian/tests/test_debian.py
	swh/loader/package/debian/tests/test_tasks.py
	swh/loader/package/debian/tests/data/http_deb.debian.org/debian_pool_contrib_c_cicero_cicero_0.7.2-3.diff.gz
	swh/loader/package/debian/tests/data/http_deb.debian.org/debian_pool_contrib_c_cicero_cicero_0.7.2-3.dsc
	swh/loader/package/debian/tests/data/http_deb.debian.org/debian_pool_contrib_c_cicero_cicero_0.7.2-4.diff.gz
	swh/loader/package/debian/tests/data/http_deb.debian.org/debian_pool_contrib_c_cicero_cicero_0.7.2-4.dsc
	swh/loader/package/debian/tests/data/http_deb.debian.org/debian_pool_contrib_c_cicero_cicero_0.7.2.orig.tar.gz
	swh/loader/package/debian/tests/data/http_deb.debian.org/onefile.txt
	swh/loader/package/deposit/__init__.py
	swh/loader/package/deposit/loader.py
	swh/loader/package/deposit/tasks.py
	swh/loader/package/deposit/tests/__init__.py
	swh/loader/package/deposit/tests/test_deposit.py
	swh/loader/package/deposit/tests/test_tasks.py
	swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/1_private_666_meta
	swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/1_private_666_raw
	swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello-2.10.zip
	swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.10.json
	-swh/loader/package/functional/__init__.py
	-swh/loader/package/functional/loader.py
	-swh/loader/package/functional/tasks.py
	-swh/loader/package/functional/tests/data/https_example.com/file.txt
	-swh/loader/package/functional/tests/data/https_fail.com/truncated-archive.tgz
	-swh/loader/package/functional/tests/data/https_github.com/owner-1_repository-1_revision-1.tgz
	-swh/loader/package/functional/tests/data/https_github.com/owner-2_repository-1_revision-1.tgz
	-swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json
	-swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json
	-swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1
	+swh/loader/package/nixguix/__init__.py
	+swh/loader/package/nixguix/loader.py
	+swh/loader/package/nixguix/tasks.py
	+swh/loader/package/nixguix/tests/data/https_example.com/file.txt
	+swh/loader/package/nixguix/tests/data/https_fail.com/truncated-archive.tgz
	+swh/loader/package/nixguix/tests/data/https_github.com/owner-1_repository-1_revision-1.tgz
	+swh/loader/package/nixguix/tests/data/https_github.com/owner-2_repository-1_revision-1.tgz
	+swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json
	+swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json
	+swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1
	swh/loader/package/npm/__init__.py
	swh/loader/package/npm/loader.py
	swh/loader/package/npm/tasks.py
	swh/loader/package/npm/tests/__init__.py
	swh/loader/package/npm/tests/test_npm.py
	swh/loader/package/npm/tests/test_tasks.py
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/@aller_shared_-_shared-0.1.0.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/@aller_shared_-_shared-0.1.1-alpha.14.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/jammit-express_-_jammit-express-0.0.1.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/nativescript-telerik-analytics_-_nativescript-telerik-analytics-1.0.0.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/org_-_org-0.0.2.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/org_-_org-0.0.3.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/org_-_org-0.0.4.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/org_-_org-0.0.5.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/org_-_org-0.1.0.tgz
	swh/loader/package/npm/tests/data/https_registry.npmjs.org/org_-_org-0.2.0.tgz
	swh/loader/package/npm/tests/data/https_replicate.npmjs.com/@aller_shared
	swh/loader/package/npm/tests/data/https_replicate.npmjs.com/catify
	swh/loader/package/npm/tests/data/https_replicate.npmjs.com/jammit-express
	swh/loader/package/npm/tests/data/https_replicate.npmjs.com/jammit-no-time
	swh/loader/package/npm/tests/data/https_replicate.npmjs.com/nativescript-telerik-analytics
	swh/loader/package/npm/tests/data/https_replicate.npmjs.com/org
	swh/loader/package/npm/tests/data/https_replicate.npmjs.com/org_visit1
	swh/loader/package/pypi/__init__.py
	swh/loader/package/pypi/loader.py
	swh/loader/package/pypi/tasks.py
	swh/loader/package/pypi/tests/__init__.py
	swh/loader/package/pypi/tests/test_pypi.py
	swh/loader/package/pypi/tests/test_tasks.py
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/0805nexter-1.1.0.tar.gz
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/0805nexter-1.1.0.zip
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/0805nexter-1.2.0.zip
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/0805nexter-1.3.0.zip
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/0805nexter-1.4.0.zip
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/nexter-1.1.0.tar.gz
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/nexter-1.1.0.zip
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/packages_70_97_c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1_0805nexter-1.3.0.zip
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/packages_86_10_c9555ec63106153aaaad753a281ff47f4ac79e980ff7f5d740d6649cd56a_upymenu-0.0.1.tar.gz
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/packages_c4_a0_4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4_0805nexter-1.2.0.zip
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/packages_c4_a0_4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4_0805nexter-1.2.0.zip_visit1
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/packages_ec_65_c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d_0805nexter-1.1.0.zip
	swh/loader/package/pypi/tests/data/https_files.pythonhosted.org/packages_ec_65_c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d_0805nexter-1.1.0.zip_visit1
	swh/loader/package/pypi/tests/data/https_pypi.org/pypi_0805nexter_json
	swh/loader/package/pypi/tests/data/https_pypi.org/pypi_0805nexter_json_visit1
	swh/loader/package/pypi/tests/data/https_pypi.org/pypi_nexter_json
	swh/loader/package/pypi/tests/data/https_pypi.org/pypi_upymenu_json
	swh/loader/package/tests/__init__.py
	swh/loader/package/tests/common.py
	swh/loader/package/tests/test_common.py
	swh/loader/package/tests/test_conftest.py
	swh/loader/package/tests/test_loader.py
	swh/loader/package/tests/test_utils.py
	swh/loader/tests/__init__.py
	swh/loader/tests/conftest.py
	swh/loader/tests/test_cli.py
	\ No newline at end of file
	diff --git a/swh.loader.core.egg-info/entry_points.txt b/swh.loader.core.egg-info/entry_points.txt
	index d393358..3f5fe42 100644
	--- a/swh.loader.core.egg-info/entry_points.txt
	+++ b/swh.loader.core.egg-info/entry_points.txt
	@@ -1,12 +1,12 @@

	[swh.workers]
	loader.archive=swh.loader.package.archive:register
	loader.cran=swh.loader.package.cran:register
	loader.debian=swh.loader.package.debian:register
	loader.deposit=swh.loader.package.deposit:register
	- loader.functional=swh.loader.package.functional:register
	+ loader.nixguix=swh.loader.package.nixguix:register
	loader.npm=swh.loader.package.npm:register
	loader.pypi=swh.loader.package.pypi:register
	[swh.cli.subcommands]
	loader=swh.loader.cli:loader

	\ No newline at end of file
	diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py
	index c0177da..cb31f24 100644
	--- a/swh/loader/core/loader.py
	+++ b/swh/loader/core/loader.py
	@@ -1,409 +1,420 @@
	# Copyright (C) 2015-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import datetime
	import hashlib
	import logging
	import os

	from abc import ABCMeta, abstractmethod
	from typing import Any, Dict, Iterable, Optional, Tuple, Union

	from swh.core import config
	from swh.model.model import (
	BaseContent, Content, SkippedContent, Directory, Origin, Revision,
	- Release, Snapshot)
	+ Release, Sha1Git, Snapshot
	+)
	from swh.storage import get_storage


	class BaseLoader(config.SWHConfig, metaclass=ABCMeta):
	"""Mixin base class for loader.

	To use this class, you must:

	- inherit from this class
	- and implement the @abstractmethod methods:

	- :func:`prepare`: First step executed by the loader to prepare some
	state needed by the `func`:load method.

	- :func:`get_origin`: Retrieve the origin that is currently being loaded.

	- :func:`fetch_data`: Fetch the data is actually the method to implement
	to compute data to inject in swh (through the store_data method)

	- :func:`store_data`: Store data fetched.

	- :func:`visit_status`: Explicit status of the visit ('partial' or
	'full')

	- :func:`load_status`: Explicit status of the loading, for use by the
	scheduler (eventful/uneventful/temporary failure/permanent failure).

	- :func:`cleanup`: Last step executed by the loader.

	The entry point for the resulting loader is :func:`load`.

	You can take a look at some example classes:

	- :class:`BaseSvnLoader`

	"""
	CONFIG_BASE_FILENAME = None # type: Optional[str]

	DEFAULT_CONFIG = {
	'storage': ('dict', {
	'cls': 'remote',
	'args': {
	'url': 'http://localhost:5002/',
	}
	}),

	'max_content_size': ('int', 100 * 1024 * 1024),
	'save_data': ('bool', False),
	'save_data_path': ('str', ''),

	} # type: Dict[str, Tuple[str, Any]]

	ADDITIONAL_CONFIG = {} # type: Dict[str, Tuple[str, Any]]

	def __init__(self, logging_class: Optional[str] = None,
	config: Dict[str, Any] = {}):
	if config:
	self.config = config
	else:
	self.config = self.parse_config_file(
	additional_configs=[self.ADDITIONAL_CONFIG])

	self.storage = get_storage(**self.config['storage'])

	if logging_class is None:
	logging_class = '%s.%s' % (self.__class__.__module__,
	self.__class__.__name__)
	self.log = logging.getLogger(logging_class)

	_log = logging.getLogger('requests.packages.urllib3.connectionpool')
	_log.setLevel(logging.WARN)

	self.max_content_size = self.config['max_content_size']

	# possibly overridden in self.prepare method
	self.visit_date: Optional[Union[str, datetime.datetime]] = None

	self.origin: Optional[Origin] = None

	if not hasattr(self, 'visit_type'):
	self.visit_type: Optional[str] = None

	self.origin_metadata: Dict[str, Any] = {}

	# Make sure the config is sane
	save_data = self.config.get('save_data')
	if save_data:
	path = self.config['save_data_path']
	os.stat(path)
	if not os.access(path, os.R_OK \| os.W_OK):
	raise PermissionError("Permission denied: %r" % path)

	def save_data(self) -> None:
	"""Save the data associated to the current load"""
	raise NotImplementedError

	def get_save_data_path(self) -> str:
	"""The path to which we archive the loader's raw data"""
	if not hasattr(self, '__save_data_path'):
	year = str(self.visit_date.year) # type: ignore

	assert self.origin
	url = self.origin.url.encode('utf-8')
	origin_url_hash = hashlib.sha1(url).hexdigest()

	path = '%s/sha1:%s/%s/%s' % (
	self.config['save_data_path'],
	origin_url_hash[0:2],
	origin_url_hash,
	year,
	)

	os.makedirs(path, exist_ok=True)
	self.__save_data_path = path

	return self.__save_data_path

	def flush(self) -> None:
	"""Flush any potential dangling data not sent to swh-storage.

	Bypass the maybe_load_* methods which awaits threshold reached
	signal. We actually want to store those as we are done
	loading.

	"""
	if hasattr(self.storage, 'flush'):
	self.storage.flush()

	@abstractmethod
	def cleanup(self) -> None:
	"""Last step executed by the loader.

	"""
	pass

	@abstractmethod
	def prepare_origin_visit(self, args, *kwargs) -> None:
	"""First step executed by the loader to prepare origin and visit
	references. Set/update self.origin, and
	optionally self.origin_url, self.visit_date.

	"""
	pass

	def _store_origin_visit(self) -> None:
	"""Store origin and visit references. Sets the self.visit references.

	"""
	assert self.origin
	self.storage.origin_add_one(self.origin)

	if not self.visit_date: # now as default visit_date if not provided
	self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc)
	self.visit = self.storage.origin_visit_add(
	self.origin.url, self.visit_date, self.visit_type)

	@abstractmethod
	def prepare(self, args, *kwargs) -> None:
	"""Second step executed by the loader to prepare some state needed by
	the loader.

	"""
	pass

	def get_origin(self) -> Origin:
	"""Get the origin that is currently being loaded.
	self.origin should be set in :func:`prepare_origin`

	Returns:
	dict: an origin ready to be sent to storage by
	:func:`origin_add_one`.
	"""
	assert self.origin
	return self.origin

	@abstractmethod
	def fetch_data(self) -> bool:
	"""Fetch the data from the source the loader is currently loading
	(ex: git/hg/svn/... repository).

	Returns:
	a value that is interpreted as a boolean. If True, fetch_data needs
	to be called again to complete loading.

	"""
	pass

	@abstractmethod
	def store_data(self):
	"""Store fetched data in the database.

	Should call the :func:`maybe_load_xyz` methods, which handle the
	bundles sent to storage, rather than send directly.
	"""
	pass

	def store_metadata(self) -> None:
	"""Store fetched metadata in the database.

	For more information, see implementation in :class:`DepositLoader`.
	"""
	pass

	def load_status(self) -> Dict[str, str]:
	"""Detailed loading status.

	Defaults to logging an eventful load.

	Returns: a dictionary that is eventually passed back as the task's
	result to the scheduler, allowing tuning of the task recurrence
	mechanism.
	"""
	return {
	'status': 'eventful',
	}

	def post_load(self, success: bool = True) -> None:
	"""Permit the loader to do some additional actions according to status
	after the loading is done. The flag success indicates the
	loading's status.

	Defaults to doing nothing.

	This is up to the implementer of this method to make sure this
	does not break.

	Args:
	success (bool): the success status of the loading

	"""
	pass

	def visit_status(self) -> str:
	"""Detailed visit status.

	Defaults to logging a full visit.
	"""
	return 'full'

	+ def get_snapshot_id(self) -> Optional[Sha1Git]:
	+ """Get the snapshot id that needs to be loaded"""
	+ raise NotImplementedError
	+
	def pre_cleanup(self) -> None:
	"""As a first step, will try and check for dangling data to cleanup.
	This should do its best to avoid raising issues.

	"""
	pass

	def load(self, args, *kwargs) -> Dict[str, str]:
	r"""Loading logic for the loader to follow:

	- 1. Call :meth:`prepare_origin_visit` to prepare the
	origin and visit we will associate loading data to
	- 2. Store the actual ``origin_visit`` to storage
	- 3. Call :meth:`prepare` to prepare any eventual state
	- 4. Call :meth:`get_origin` to get the origin we work with and store

	- while True:

	- 5. Call :meth:`fetch_data` to fetch the data to store
	- 6. Call :meth:`store_data` to store the data

	- 7. Call :meth:`cleanup` to clean up any eventual state put in place
	in :meth:`prepare` method.

	"""
	try:
	self.pre_cleanup()
	except Exception:
	msg = 'Cleaning up dangling data failed! Continue loading.'
	self.log.warning(msg)

	self.prepare_origin_visit(args, *kwargs)
	self._store_origin_visit()

	assert self.origin

	try:
	self.prepare(args, *kwargs)

	while True:
	more_data_to_fetch = self.fetch_data()
	self.store_data()
	if not more_data_to_fetch:
	break

	self.store_metadata()
	self.storage.origin_visit_update(
	- self.origin.url, self.visit.visit, self.visit_status()
	+ self.origin.url, self.visit.visit, self.visit_status(),
	+ snapshot=self.get_snapshot_id()
	)
	self.post_load()
	except Exception:
	self.log.exception('Loading failure, updating to `partial` status',
	extra={
	'swh_task_args': args,
	'swh_task_kwargs': kwargs,
	})
	self.storage.origin_visit_update(
	- self.origin.url, self.visit.visit, 'partial'
	+ self.origin.url, self.visit.visit, 'partial',
	+ snapshot=self.get_snapshot_id()
	)
	self.post_load(success=False)
	return {'status': 'failed'}
	finally:
	self.flush()
	self.cleanup()

	return self.load_status()


	class DVCSLoader(BaseLoader):
	"""This base class is a pattern for dvcs loaders (e.g. git, mercurial).

	Those loaders are able to load all the data in one go. For example, the
	loader defined in swh-loader-git :class:`BulkUpdater`.

	For other loaders (stateful one, (e.g :class:`SWHSvnLoader`),
	inherit directly from :class:`BaseLoader`.

	"""
	ADDITIONAL_CONFIG = {} # type: Dict[str, Tuple[str, Any]]

	def cleanup(self) -> None:
	"""Clean up an eventual state installed for computations."""
	pass

	def has_contents(self) -> bool:
	"""Checks whether we need to load contents"""
	return True

	def get_contents(self) -> Iterable[BaseContent]:
	"""Get the contents that need to be loaded"""
	raise NotImplementedError

	def has_directories(self) -> bool:
	"""Checks whether we need to load directories"""
	return True

	def get_directories(self) -> Iterable[Directory]:
	"""Get the directories that need to be loaded"""
	raise NotImplementedError

	def has_revisions(self) -> bool:
	"""Checks whether we need to load revisions"""
	return True

	def get_revisions(self) -> Iterable[Revision]:
	"""Get the revisions that need to be loaded"""
	raise NotImplementedError

	def has_releases(self) -> bool:
	"""Checks whether we need to load releases"""
	return True

	def get_releases(self) -> Iterable[Release]:
	"""Get the releases that need to be loaded"""
	raise NotImplementedError

	def get_snapshot(self) -> Snapshot:
	"""Get the snapshot that needs to be loaded"""
	raise NotImplementedError

	+ def get_snapshot_id(self) -> Optional[Sha1Git]:
	+ snapshot = self.get_snapshot()
	+ return snapshot.id if snapshot else None
	+
	def eventful(self) -> bool:
	"""Whether the load was eventful"""
	raise NotImplementedError

	def store_data(self) -> None:
	assert self.origin
	if self.config['save_data']:
	self.save_data()

	if self.has_contents():
	contents = []
	skipped_contents = []
	for obj in self.get_contents():
	if isinstance(obj, Content):
	contents.append(obj)
	elif isinstance(obj, SkippedContent):
	skipped_contents.append(obj)
	else:
	raise TypeError(f'Unexpected content type: {obj}')
	self.storage.skipped_content_add(skipped_contents)
	self.storage.content_add(contents)
	if self.has_directories():
	self.storage.directory_add(self.get_directories())
	if self.has_revisions():
	self.storage.revision_add(self.get_revisions())
	if self.has_releases():
	self.storage.release_add(self.get_releases())
	self.flush() # to ensure the snapshot targets existing objects
	snapshot = self.get_snapshot()
	self.storage.snapshot_add([snapshot])
	self.storage.origin_visit_update(
	self.origin.url, self.visit.visit, snapshot=snapshot.id)
	self.flush()
	diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py
	index 4eb1602..4761f98 100644
	--- a/swh/loader/core/tests/test_loader.py
	+++ b/swh/loader/core/tests/test_loader.py
	@@ -1,146 +1,149 @@
	# Copyright (C) 2018-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import datetime
	import hashlib
	import logging
	import pytest

	from swh.model.model import Origin

	from swh.loader.core.loader import BaseLoader, DVCSLoader


	class DummyLoader:
	def cleanup(self):
	pass

	def prepare(self, args, *kwargs):
	pass

	def fetch_data(self):
	pass

	def store_data(self):
	pass

	+ def get_snapshot_id(self):
	+ return None
	+
	def prepare_origin_visit(self, args, *kwargs):
	origin = Origin(url='some-url')
	self.origin = origin
	self.origin_url = origin.url
	self.visit_date = datetime.datetime.utcnow()
	self.visit_type = 'git'
	origin_url = self.storage.origin_add_one(origin)
	self.visit = self.storage.origin_visit_add(
	origin_url, self.visit_date, self.visit_type)


	class DummyDVCSLoader(DummyLoader, DVCSLoader):
	"""Unbuffered loader will send directly to storage new data

	"""
	def parse_config_file(self, args, *kwargs):
	return {
	'max_content_size': 100 * 1024 * 1024,
	'storage': {
	'cls': 'pipeline',
	'steps': [
	{
	'cls': 'retry',
	},
	{
	'cls': 'filter',
	},
	{
	'cls': 'memory',
	},
	]
	},
	}


	class DummyBaseLoader(DummyLoader, BaseLoader):
	"""Buffered loader will send new data when threshold is reached

	"""
	def parse_config_file(self, args, *kwargs):
	return {
	'max_content_size': 100 * 1024 * 1024,
	'storage': {
	'cls': 'pipeline',
	'steps': [
	{
	'cls': 'retry',
	},
	{
	'cls': 'filter',
	},
	{
	'cls': 'buffer',
	'min_batch_size': {
	'content': 2,
	'content_bytes': 8,
	'directory': 2,
	'revision': 2,
	'release': 2,
	},
	},
	{
	'cls': 'memory',
	},
	]
	},
	}


	def test_base_loader():
	loader = DummyBaseLoader()
	result = loader.load()

	assert result == {'status': 'eventful'}


	def test_dvcs_loader():
	loader = DummyDVCSLoader()
	result = loader.load()
	assert result == {'status': 'eventful'}


	def test_loader_logger_default_name():
	loader = DummyBaseLoader()
	assert isinstance(loader.log, logging.Logger)
	assert loader.log.name == \
	'swh.loader.core.tests.test_loader.DummyBaseLoader'

	loader = DummyDVCSLoader()
	assert isinstance(loader.log, logging.Logger)
	assert loader.log.name == \
	'swh.loader.core.tests.test_loader.DummyDVCSLoader'


	def test_loader_logger_with_name():
	loader = DummyBaseLoader('some.logger.name')
	assert isinstance(loader.log, logging.Logger)
	assert loader.log.name == \
	'some.logger.name'


	@pytest.mark.fs
	def test_loader_save_data_path(tmp_path):
	loader = DummyBaseLoader('some.logger.name.1')
	url = 'http://bitbucket.org/something'
	loader.origin = Origin(url=url)
	loader.visit_date = datetime.datetime(year=2019, month=10, day=1)
	loader.config = {
	'save_data_path': tmp_path,
	}

	hash_url = hashlib.sha1(url.encode('utf-8')).hexdigest()
	expected_save_path = '%s/sha1:%s/%s/2019' % (
	str(tmp_path), hash_url[0:2], hash_url
	)

	save_path = loader.get_save_data_path()
	assert save_path == expected_save_path
	diff --git a/swh/loader/package/functional/__init__.py b/swh/loader/package/nixguix/__init__.py
	similarity index 85%
	rename from swh/loader/package/functional/__init__.py
	rename to swh/loader/package/nixguix/__init__.py
	index 3bf7da1..3378d59 100644
	--- a/swh/loader/package/functional/__init__.py
	+++ b/swh/loader/package/nixguix/__init__.py
	@@ -1,16 +1,16 @@
	# Copyright (C) 2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information


	from typing import Any, Mapping


	def register() -> Mapping[str, Any]:
	"""Register the current worker module's definition"""
	- from .loader import FunctionalLoader
	+ from .loader import NixGuixLoader
	return {
	'task_modules': [f'{__name__}.tasks'],
	- 'loader': FunctionalLoader,
	+ 'loader': NixGuixLoader,
	}
	diff --git a/swh/loader/package/functional/loader.py b/swh/loader/package/nixguix/loader.py
	similarity index 98%
	rename from swh/loader/package/functional/loader.py
	rename to swh/loader/package/nixguix/loader.py
	index 381f190..54c94ec 100644
	--- a/swh/loader/package/functional/loader.py
	+++ b/swh/loader/package/nixguix/loader.py
	@@ -1,115 +1,115 @@
	# Copyright (C) 2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import json
	import requests

	from typing import Dict, Optional, Any, Mapping

	from swh.model import hashutil

	from swh.model.model import (
	Sha1Git, Revision, RevisionType
	)

	from swh.loader.package.utils import EMPTY_AUTHOR

	from swh.loader.package.loader import PackageLoader


	def retrieve_sources(url: str) -> Dict[str, Any]:
	response = requests.get(url,
	allow_redirects=True)
	if response.status_code != 200:
	raise ValueError("Got %d HTTP code on %s",
	response.status_code, url)

	return json.loads(response.content.decode('utf-8'))


	-class FunctionalLoader(PackageLoader):
	+class NixGuixLoader(PackageLoader):
	"""Load sources from a sources.json file. This loader is used to load
	sources used by functional package manager (eg. Nix and Guix).

	"""
	- visit_type = 'functional'
	+ visit_type = 'nixguix'

	def __init__(self, url):
	super().__init__(url=url)
	s = retrieve_sources(url)
	self.sources = s['sources']
	self.provider_url = url
	# The revision used to create the sources.json file. For Nix,
	# this revision belongs to the github.com/nixos/nixpkgs
	# repository
	self.revision = s['revision']

	# Note: this could be renamed get_artifacts in the PackageLoader
	# base class.
	def get_versions(self):
	# TODO: try all mirrors and not only the first one. A source
	# can be fetched from several urls, called mirrors. We
	# currently only use the first one, but if the first one
	# fails, we should try the second one and so on.
	return [s['url'][0] for s in self.sources]

	# Note: this could be renamed get_artifact_info in the PackageLoader
	# base class.
	def get_package_info(self, source):
	# TODO: we need to provide the sha256 of the source also
	yield source, {'url': source, 'raw': {'url': source}}

	def resolve_revision_from(
	self, known_artifacts: Dict, artifact_metadata: Dict) \
	-> Optional[bytes]:
	for rev_id, known_artifact in known_artifacts.items():
	known_url = known_artifact['extrinsic']['raw']['url']
	if artifact_metadata['url'] == known_url:
	return rev_id
	return None

	def extra_branches(self) -> Dict[bytes, Mapping[str, Any]]:
	"""We add a branch to the snapshot called 'evaluation' pointing to the
	revision used to generate the sources.json file. This revision
	is specified in the sources.json file itself. For the nixpkgs
	origin, this revision is coming from the
	github.com/nixos/nixpkgs repository.

	Note this repository is not loaded explicitly. So, this
	pointer can target a nonexistent revision for a time. However,
	the github and gnu loaders are supposed to load this revision
	and should create the revision pointed by this branch.

	This branch can be used to identify the snapshot associated to
	a Nix/Guix evaluation.

	"""
	return {
	b'evaluation': {
	'target_type': 'revision',
	'target': hashutil.hash_to_bytes(self.revision)
	}
	}

	def build_revision(self, a_metadata: Dict, uncompressed_path: str,
	directory: Sha1Git) -> Optional[Revision]:
	return Revision(
	type=RevisionType.TAR,
	message=b'',
	author=EMPTY_AUTHOR,
	date=None,
	committer=EMPTY_AUTHOR,
	committer_date=None,
	parents=[],
	directory=directory,
	synthetic=True,
	metadata={
	'extrinsic': {
	'provider': self.provider_url,
	'when': self.visit_date.isoformat(),
	'raw': a_metadata,
	},
	}
	)
	diff --git a/swh/loader/package/functional/tasks.py b/swh/loader/package/nixguix/tasks.py
	similarity index 63%
	rename from swh/loader/package/functional/tasks.py
	rename to swh/loader/package/nixguix/tasks.py
	index 35b47e5..75517a6 100644
	--- a/swh/loader/package/functional/tasks.py
	+++ b/swh/loader/package/nixguix/tasks.py
	@@ -1,16 +1,16 @@
	# Copyright (C) 2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	from celery import shared_task

	-from swh.loader.package.functional.loader import (
	- FunctionalLoader
	+from swh.loader.package.nixguix.loader import (
	+ NixGuixLoader
	)


	-@shared_task(name=__name__ + '.LoadFunctional')
	-def load_functional(*, url=None):
	+@shared_task(name=__name__ + '.LoadNixguix')
	+def load_nixguix(*, url=None):
	"""Load functional (e.g. guix/nix) package"""
	- return FunctionalLoader(url).load()
	+ return NixGuixLoader(url).load()
	diff --git a/swh/loader/package/functional/tests/data/https_example.com/file.txt b/swh/loader/package/nixguix/tests/data/https_example.com/file.txt
	similarity index 100%
	rename from swh/loader/package/functional/tests/data/https_example.com/file.txt
	rename to swh/loader/package/nixguix/tests/data/https_example.com/file.txt
	diff --git a/swh/loader/package/functional/tests/data/https_fail.com/truncated-archive.tgz b/swh/loader/package/nixguix/tests/data/https_fail.com/truncated-archive.tgz
	similarity index 100%
	rename from swh/loader/package/functional/tests/data/https_fail.com/truncated-archive.tgz
	rename to swh/loader/package/nixguix/tests/data/https_fail.com/truncated-archive.tgz
	diff --git a/swh/loader/package/functional/tests/data/https_github.com/owner-1_repository-1_revision-1.tgz b/swh/loader/package/nixguix/tests/data/https_github.com/owner-1_repository-1_revision-1.tgz
	similarity index 100%
	rename from swh/loader/package/functional/tests/data/https_github.com/owner-1_repository-1_revision-1.tgz
	rename to swh/loader/package/nixguix/tests/data/https_github.com/owner-1_repository-1_revision-1.tgz
	diff --git a/swh/loader/package/functional/tests/data/https_github.com/owner-2_repository-1_revision-1.tgz b/swh/loader/package/nixguix/tests/data/https_github.com/owner-2_repository-1_revision-1.tgz
	similarity index 100%
	rename from swh/loader/package/functional/tests/data/https_github.com/owner-2_repository-1_revision-1.tgz
	rename to swh/loader/package/nixguix/tests/data/https_github.com/owner-2_repository-1_revision-1.tgz
	diff --git a/swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json
	similarity index 100%
	rename from swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json
	rename to swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources-EOFError.json
	diff --git a/swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json
	similarity index 100%
	rename from swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json
	rename to swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json
	diff --git a/swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1 b/swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1
	similarity index 100%
	rename from swh/loader/package/functional/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1
	rename to swh/loader/package/nixguix/tests/data/https_nix-community.github.io/nixpkgs-swh_sources.json_visit1
	diff --git a/swh/loader/tests/test_cli.py b/swh/loader/tests/test_cli.py
	index f50b60c..9e36a4b 100644
	--- a/swh/loader/tests/test_cli.py
	+++ b/swh/loader/tests/test_cli.py
	@@ -1,113 +1,113 @@
	# Copyright (C) 2019 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import pytest


	from swh.loader.cli import run, list, get_loader, SUPPORTED_LOADERS
	from swh.loader.package.loader import PackageLoader

	from click.testing import CliRunner


	def test_get_loader_wrong_input(swh_config):
	"""Unsupported loader should raise

	"""
	loader_type = 'unknown'
	assert loader_type not in SUPPORTED_LOADERS
	with pytest.raises(ValueError, match='Invalid loader'):
	get_loader(loader_type, url='db-url')


	def test_get_loader(swh_config):
	"""Instantiating a supported loader should be ok

	"""
	loader_input = {
	'archive': {
	'url': 'some-url',
	'artifacts': [],
	},
	'debian': {
	'url': 'some-url',
	'date': 'something',
	'packages': [],
	},
	'deposit': {
	'url': 'some-url',
	'deposit_id': 1,
	},
	'npm': {
	'url': 'https://www.npmjs.com/package/onepackage',
	},
	'pypi': {
	'url': 'some-url',
	},
	}
	for loader_type, kwargs in loader_input.items():
	loader = get_loader(loader_type, **kwargs)
	assert isinstance(loader, PackageLoader)


	def test_run_help(swh_config):
	"""Help message should be ok

	"""
	runner = CliRunner()
	result = runner.invoke(run, ['-h'])

	assert result.exit_code == 0
	- expected_help_msg = """Usage: run [OPTIONS] [archive\|cran\|debian\|deposit\|functional\|npm\|pypi] URL
	+ expected_help_msg = """Usage: run [OPTIONS] [archive\|cran\|debian\|deposit\|nixguix\|npm\|pypi] URL
	[OPTIONS]...

	Ingest with loader <type> the origin located at <url>

	Options:
	-h, --help Show this message and exit.
	""" # noqa

	assert result.output.startswith(expected_help_msg)


	def test_run_pypi(mocker, swh_config):
	"""Triggering a load should be ok

	"""
	mock_loader = mocker.patch('swh.loader.package.pypi.loader.PyPILoader')
	runner = CliRunner()
	result = runner.invoke(run, ['pypi', 'https://some-url'])
	assert result.exit_code == 0
	mock_loader.assert_called_once_with(url='https://some-url') # constructor


	def test_list_help(mocker, swh_config):
	"""Triggering a load should be ok

	"""
	runner = CliRunner()
	result = runner.invoke(list, ['--help'])
	assert result.exit_code == 0
	- expected_help_msg = """Usage: list [OPTIONS] [[all\|archive\|cran\|debian\|deposit\|functional\|npm\|pypi]]
	+ expected_help_msg = """Usage: list [OPTIONS] [[all\|archive\|cran\|debian\|deposit\|nixguix\|npm\|pypi]]

	List supported loaders and optionally their arguments

	Options:
	-h, --help Show this message and exit.
	""" # noqa
	assert result.output.startswith(expected_help_msg)


	def test_list_help_npm(mocker, swh_config):
	"""Triggering a load should be ok

	"""
	runner = CliRunner()
	result = runner.invoke(list, ['npm'])
	assert result.exit_code == 0
	expected_help_msg = '''Loader: Load npm origin's artifact releases into swh archive.
	signature: (url: str)
	''' # noqa
	assert result.output.startswith(expected_help_msg)
	diff --git a/version.txt b/version.txt
	index e61a032..4428541 100644
	--- a/version.txt
	+++ b/version.txt
	@@ -1 +1 @@
	-v0.0.84-0-g81202fb
	\ No newline at end of file
	+v0.0.85-0-g4e86ca6
	\ No newline at end of file

File Metadata

Mime Type: text/x-diff
Expires: Mon, Aug 25, 5:35 PM (1 w, 3 h ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3443744

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions