Page MenuHomeSoftware Heritage
Differential D7492

docker: add librdkakfa-dev so confluent-kafka in pip installable on non-x86 platforms
ClosedPublic
Actions

Authored by douardda on Apr 4 2022, 2:05 PM.

Details

Reviewers
anlambert
Group Reviewers
Reviewers
Commits
rDENV995e90739c3d: docker: add librdkakfa-dev so confluent-kafka in pip installable on non-x86…
Summary

docker: make yarn run in the local copy of the swh-web source code

to prevent permission issues.

Diff Detail

Repository
rDENV Development environment
Branch
master
Lint
Lint Skipped
Unit
Unit Tests Skipped
Build Status
Buildable 28086
Build 43988: arc lint + arc unit

Revision Contents
Changeset List

PathSize
docker/
1 line
services/
swh-web/
10 lines
utils/
22 lines

Diff 27174

View Options

docker/Dockerfile

Show All 27 Lines def task_dict(self, origin_type, origin_url, **kwargs):
This is overridden from the lister_base as more information is This is overridden from the lister_base as more information is
needed for the ingestion task creation. needed for the ingestion task creation.
This creates tasks with args and kwargs set, for example: This creates tasks with args and kwargs set, for example:
.. code-block:: python .. code-block:: python
args: ['https://ftp.gnu.org/gnu/3dldf/'] args:
kwargs: { kwargs: {
'tarballs': [{ 'url': 'https://ftp.gnu.org/gnu/3dldf/',
'archive': 'https://...', 'artifacts': [{
'time': 1071002600, 'url': 'https://...',
'length': 128}, 'time': '2003-12-09T21:43:20+00:00',
'length': 128,
'version': '1.0.1',
'filename': 'something-1.0.1.tar.gz',
},
... ...
]} ]
}
""" """
tarballs = self.gnu_tree.artifacts[origin_url] artifacts = self.gnu_tree.artifacts[origin_url]
return utils.create_task_dict( return utils.create_task_dict(
'load-%s' % origin_type, 'load-%s' % origin_type,
kwargs.get('policy', 'oneshot'), kwargs.get('policy', 'oneshot'),
origin_url, url=origin_url,
tarballs=tarballs) artifacts=artifacts)
def safely_issue_request(self, identifier): def safely_issue_request(self, identifier):
"""Bypass the implementation. It's now the GNUTree which deals with """Bypass the implementation. It's now the GNUTree which deals with
querying the gnu mirror. querying the gnu mirror.
As an implementation detail, we cannot change simply the base As an implementation detail, we cannot change simply the base
SimpleLister as other implementation still uses it. This shall be part SimpleLister as other implementation still uses it. This shall be part
of another refactoring pass. of another refactoring pass.
""" """
return None return None
def list_packages(self, response): def list_packages(self, response):
"""List the actual gnu origins (package name) with their name, url and """List the actual gnu origins (package name) with their name, url and
associated tarballs. associated tarballs.
Args: Args:
response: Unused response: Unused
Returns: Returns:
List of packages name, url, last modification time List of packages name, url, last modification time
.. code-block:: python .. code-block:: python
[ [
{'name': '3dldf', {'name': '3dldf',
'url': 'https://ftp.gnu.org/gnu/3dldf/', 'url': 'https://ftp.gnu.org/gnu/3dldf/',
'time_modified': 1071002600}, 'time_modified': '2003-12-09T20:43:20+00:00'},
{'name': '8sync', {'name': '8sync',
'url': 'https://ftp.gnu.org/gnu/8sync/', 'url': 'https://ftp.gnu.org/gnu/8sync/',
'time_modified': 1480991830}, 'time_modified': '2016-12-06T02:37:10+00:00'},
... ...
] ]
""" """
return list(self.gnu_tree.projects.values()) return list(self.gnu_tree.projects.values())
def get_model_from_repo(self, repo): def get_model_from_repo(self, repo):
"""Transform from repository representation to model """Transform from repository representation to model
""" """
return { return {
'uid': repo['name'], 'uid': repo['name'],
'name': repo['name'], 'name': repo['name'],
'full_name': repo['name'], 'full_name': repo['name'],
'html_url': repo['url'], 'html_url': repo['url'],
'origin_url': repo['url'], 'origin_url': repo['url'],
'time_last_updated': int(repo['time_modified']), 'time_last_updated': repo['time_modified'],
'origin_type': 'tar', 'origin_type': 'tar',
} }
View Options

docker/services/swh-web/entrypoint.sh

Show All 27 Lines def task_dict(self, origin_type, origin_url, **kwargs):
This is overridden from the lister_base as more information is This is overridden from the lister_base as more information is
needed for the ingestion task creation. needed for the ingestion task creation.
This creates tasks with args and kwargs set, for example: This creates tasks with args and kwargs set, for example:
.. code-block:: python .. code-block:: python
args: ['https://ftp.gnu.org/gnu/3dldf/'] args:
kwargs: { kwargs: {
'tarballs': [{ 'url': 'https://ftp.gnu.org/gnu/3dldf/',
'archive': 'https://...', 'artifacts': [{
'time': 1071002600, 'url': 'https://...',
'length': 128}, 'time': '2003-12-09T21:43:20+00:00',
'length': 128,
'version': '1.0.1',
'filename': 'something-1.0.1.tar.gz',
},
... ...
]} ]
}
""" """
tarballs = self.gnu_tree.artifacts[origin_url] artifacts = self.gnu_tree.artifacts[origin_url]
return utils.create_task_dict( return utils.create_task_dict(
'load-%s' % origin_type, 'load-%s' % origin_type,
kwargs.get('policy', 'oneshot'), kwargs.get('policy', 'oneshot'),
origin_url, url=origin_url,
tarballs=tarballs) artifacts=artifacts)
def safely_issue_request(self, identifier): def safely_issue_request(self, identifier):
"""Bypass the implementation. It's now the GNUTree which deals with """Bypass the implementation. It's now the GNUTree which deals with
querying the gnu mirror. querying the gnu mirror.
As an implementation detail, we cannot change simply the base As an implementation detail, we cannot change simply the base
SimpleLister as other implementation still uses it. This shall be part SimpleLister as other implementation still uses it. This shall be part
of another refactoring pass. of another refactoring pass.
""" """
return None return None
def list_packages(self, response): def list_packages(self, response):
"""List the actual gnu origins (package name) with their name, url and """List the actual gnu origins (package name) with their name, url and
associated tarballs. associated tarballs.
Args: Args:
response: Unused response: Unused
Returns: Returns:
List of packages name, url, last modification time List of packages name, url, last modification time
.. code-block:: python .. code-block:: python
[ [
{'name': '3dldf', {'name': '3dldf',
'url': 'https://ftp.gnu.org/gnu/3dldf/', 'url': 'https://ftp.gnu.org/gnu/3dldf/',
'time_modified': 1071002600}, 'time_modified': '2003-12-09T20:43:20+00:00'},
{'name': '8sync', {'name': '8sync',
'url': 'https://ftp.gnu.org/gnu/8sync/', 'url': 'https://ftp.gnu.org/gnu/8sync/',
'time_modified': 1480991830}, 'time_modified': '2016-12-06T02:37:10+00:00'},
... ...
] ]
""" """
return list(self.gnu_tree.projects.values()) return list(self.gnu_tree.projects.values())
def get_model_from_repo(self, repo): def get_model_from_repo(self, repo):
"""Transform from repository representation to model """Transform from repository representation to model
""" """
return { return {
'uid': repo['name'], 'uid': repo['name'],
'name': repo['name'], 'name': repo['name'],
'full_name': repo['name'], 'full_name': repo['name'],
'html_url': repo['url'], 'html_url': repo['url'],
'origin_url': repo['url'], 'origin_url': repo['url'],
'time_last_updated': int(repo['time_modified']), 'time_last_updated': repo['time_modified'],
'origin_type': 'tar', 'origin_type': 'tar',
} }
View Options

docker/utils/pyutils.sh

Show All 27 Lines def task_dict(self, origin_type, origin_url, **kwargs):
This is overridden from the lister_base as more information is This is overridden from the lister_base as more information is
needed for the ingestion task creation. needed for the ingestion task creation.
This creates tasks with args and kwargs set, for example: This creates tasks with args and kwargs set, for example:
.. code-block:: python .. code-block:: python
args: ['https://ftp.gnu.org/gnu/3dldf/'] args:
kwargs: { kwargs: {
'tarballs': [{ 'url': 'https://ftp.gnu.org/gnu/3dldf/',
'archive': 'https://...', 'artifacts': [{
'time': 1071002600, 'url': 'https://...',
'length': 128}, 'time': '2003-12-09T21:43:20+00:00',
'length': 128,
'version': '1.0.1',
'filename': 'something-1.0.1.tar.gz',
},
... ...
]} ]
}
""" """
tarballs = self.gnu_tree.artifacts[origin_url] artifacts = self.gnu_tree.artifacts[origin_url]
return utils.create_task_dict( return utils.create_task_dict(
'load-%s' % origin_type, 'load-%s' % origin_type,
kwargs.get('policy', 'oneshot'), kwargs.get('policy', 'oneshot'),
origin_url, url=origin_url,
tarballs=tarballs) artifacts=artifacts)
def safely_issue_request(self, identifier): def safely_issue_request(self, identifier):
"""Bypass the implementation. It's now the GNUTree which deals with """Bypass the implementation. It's now the GNUTree which deals with
querying the gnu mirror. querying the gnu mirror.
As an implementation detail, we cannot change simply the base As an implementation detail, we cannot change simply the base
SimpleLister as other implementation still uses it. This shall be part SimpleLister as other implementation still uses it. This shall be part
of another refactoring pass. of another refactoring pass.
""" """
return None return None
def list_packages(self, response): def list_packages(self, response):
"""List the actual gnu origins (package name) with their name, url and """List the actual gnu origins (package name) with their name, url and
associated tarballs. associated tarballs.
Args: Args:
response: Unused response: Unused
Returns: Returns:
List of packages name, url, last modification time List of packages name, url, last modification time
.. code-block:: python .. code-block:: python
[ [
{'name': '3dldf', {'name': '3dldf',
'url': 'https://ftp.gnu.org/gnu/3dldf/', 'url': 'https://ftp.gnu.org/gnu/3dldf/',
'time_modified': 1071002600}, 'time_modified': '2003-12-09T20:43:20+00:00'},
{'name': '8sync', {'name': '8sync',
'url': 'https://ftp.gnu.org/gnu/8sync/', 'url': 'https://ftp.gnu.org/gnu/8sync/',
'time_modified': 1480991830}, 'time_modified': '2016-12-06T02:37:10+00:00'},
... ...
] ]
""" """
return list(self.gnu_tree.projects.values()) return list(self.gnu_tree.projects.values())
def get_model_from_repo(self, repo): def get_model_from_repo(self, repo):
"""Transform from repository representation to model """Transform from repository representation to model
""" """
return { return {
'uid': repo['name'], 'uid': repo['name'],
'name': repo['name'], 'name': repo['name'],
'full_name': repo['name'], 'full_name': repo['name'],
'html_url': repo['url'], 'html_url': repo['url'],
'origin_url': repo['url'], 'origin_url': repo['url'],
'time_last_updated': int(repo['time_modified']), 'time_last_updated': repo['time_modified'],
'origin_type': 'tar', 'origin_type': 'tar',
} }
About · Developer information · Legal