Differential D2147 Diff 7406 swh/lister/gnu/lister.py

Changeset View

Standalone View

swh/lister/gnu/lister.py

Show All 27 Lines	def task_dict(self, origin_type, origin_url, **kwargs):

This is overridden from the lister_base as more information is		This is overridden from the lister_base as more information is
needed for the ingestion task creation.		needed for the ingestion task creation.

This creates tasks with args and kwargs set, for example:		This creates tasks with args and kwargs set, for example:

.. code-block:: python		.. code-block:: python

args: ['https://ftp.gnu.org/gnu/3dldf/']		args:
kwargs: {		kwargs: {
'tarballs': [{		'url': 'https://ftp.gnu.org/gnu/3dldf/',
'archive': 'https://...',		'artifacts': [{
'time': 1071002600,		'url': 'https://...',
'length': 128},		'time': '2003-12-09T21:43:20+00:00',
		'length': 128,
		'version': '1.0.1',
		'filename': 'something-1.0.1.tar.gz',
		},
...		...
]}		]
		}

"""		"""
tarballs = self.gnu_tree.artifacts[origin_url]		artifacts = self.gnu_tree.artifacts[origin_url]
return utils.create_task_dict(		return utils.create_task_dict(
'load-%s' % origin_type,		'load-%s' % origin_type,
kwargs.get('policy', 'oneshot'),		kwargs.get('policy', 'oneshot'),
origin_url,		url=origin_url,
tarballs=tarballs)		artifacts=artifacts)

def safely_issue_request(self, identifier):		def safely_issue_request(self, identifier):
"""Bypass the implementation. It's now the GNUTree which deals with		"""Bypass the implementation. It's now the GNUTree which deals with
querying the gnu mirror.		querying the gnu mirror.

As an implementation detail, we cannot change simply the base		As an implementation detail, we cannot change simply the base
SimpleLister as other implementation still uses it. This shall be part		SimpleLister as other implementation still uses it. This shall be part
of another refactoring pass.		of another refactoring pass.

"""		"""
return None		return None

def list_packages(self, response):		def list_packages(self, response):
"""List the actual gnu origins (package name) with their name, url and		"""List the actual gnu origins (package name) with their name, url and
associated tarballs.		associated tarballs.

Args:		Args:
response: Unused		response: Unused

Returns:		Returns:
List of packages name, url, last modification time		List of packages name, url, last modification time

.. code-block:: python		.. code-block:: python

[		[
{'name': '3dldf',		{'name': '3dldf',
'url': 'https://ftp.gnu.org/gnu/3dldf/',		'url': 'https://ftp.gnu.org/gnu/3dldf/',
'time_modified': 1071002600},		'time_modified': '2003-12-09T20:43:20+00:00'},
{'name': '8sync',		{'name': '8sync',
'url': 'https://ftp.gnu.org/gnu/8sync/',		'url': 'https://ftp.gnu.org/gnu/8sync/',
'time_modified': 1480991830},		'time_modified': '2016-12-06T02:37:10+00:00'},
...		...
]		]

"""		"""
return list(self.gnu_tree.projects.values())		return list(self.gnu_tree.projects.values())

def get_model_from_repo(self, repo):		def get_model_from_repo(self, repo):
"""Transform from repository representation to model		"""Transform from repository representation to model

"""		"""
return {		return {
'uid': repo['name'],		'uid': repo['name'],
'name': repo['name'],		'name': repo['name'],
'full_name': repo['name'],		'full_name': repo['name'],
'html_url': repo['url'],		'html_url': repo['url'],
'origin_url': repo['url'],		'origin_url': repo['url'],
'time_last_updated': int(repo['time_modified']),		'time_last_updated': repo['time_modified'],
'origin_type': 'tar',		'origin_type': 'tar',
}		}