diff --git a/swh/lister/opam/lister.py b/swh/lister/opam/lister.py
--- a/swh/lister/opam/lister.py
+++ b/swh/lister/opam/lister.py
@@ -7,7 +7,7 @@
import logging
import os
from subprocess import PIPE, Popen, call
-from typing import Iterator, Optional
+from typing import Any, Dict, Iterator, Optional
from swh.lister.pattern import StatelessLister
from swh.scheduler.interface import SchedulerInterface
@@ -53,24 +53,12 @@
self.env = os.environ.copy()
# Opam root folder is initialized in the :meth:`get_pages` method as no
# side-effect should happen in the constructor to ease instantiation
- self.opamroot = os.path.join(opam_root, self.instance)
+ self.opam_root = opam_root
def get_pages(self) -> Iterator[PageType]:
- # Initialize the opam root directory with the opam instance data to list.
- call(
- [
- "opam",
- "init",
- "--reinit",
- "--bare",
- "--no-setup",
- "--root",
- self.opamroot,
- self.instance,
- self.url,
- ],
- env=self.env,
- )
+ # Initialize the opam root directory
+ opam_init(self.opam_root, self.instance, self.url, self.env)
+
# Actually list opam instance data
proc = Popen(
[
@@ -78,10 +66,11 @@
"list",
"--all",
"--no-switch",
+ "--safe",
"--repos",
self.instance,
"--root",
- self.opamroot,
+ self.opam_root,
"--normalise",
"--short",
],
@@ -103,9 +92,50 @@
url=url,
last_update=None,
extra_loader_arguments={
- "opam_root": self.opamroot,
+ "opam_root": self.opam_root,
"opam_instance": self.instance,
"opam_url": self.url,
"opam_package": page,
},
)
+
+
+def opam_init(opam_root: str, instance: str, url: str, env: Dict[str, Any]) -> None:
+ """Initialize an opam_root folder.
+
+ Args:
+ opam_root: The opam root folder to initialize
+ instance: Name of the opam repository to add or initialize
+ url: The associated url of the opam repository to add or initialize
+ env: The global environment to use for the opam command.
+
+ Returns:
+ None.
+
+ """
+ if not os.path.exists(opam_root) or not os.listdir(opam_root):
+ command = [
+ "opam",
+ "init",
+ "--reinit",
+ "--bare",
+ "--no-setup",
+ "--root",
+ opam_root,
+ instance,
+ url,
+ ]
+ else:
+ # The repository exists and is populated, we just add another instance in the
+ # repository. If it's already setup, it's a noop
+ command = [
+ "opam",
+ "repository",
+ "add",
+ "--root",
+ opam_root,
+ instance,
+ url,
+ ]
+ # Actually execute the command
+ call(command, env=env)
diff --git a/swh/lister/opam/tests/test_lister.py b/swh/lister/opam/tests/test_lister.py
--- a/swh/lister/opam/tests/test_lister.py
+++ b/swh/lister/opam/tests/test_lister.py
@@ -4,12 +4,13 @@
# See top-level LICENSE file for more information
import io
+import os
from tempfile import mkdtemp
from unittest.mock import MagicMock
import pytest
-from swh.lister.opam.lister import OpamLister
+from swh.lister.opam.lister import OpamLister, opam_init
module_name = "swh.lister.opam.lister"
@@ -29,6 +30,45 @@
return mock_init, mock_open
+def test_mock_init_repository_init(mock_opam, tmp_path, datadir):
+ """Initializing opam root directory with an instance should be ok
+
+ """
+ mock_init, mock_popen = mock_opam
+
+ instance = "fake"
+ instance_url = f"file://{datadir}/{instance}"
+ opam_root = str(tmp_path / "test-opam")
+ assert not os.path.exists(opam_root)
+
+ # This will initialize an opam directory with the instance
+ opam_init(opam_root, instance, instance_url, {})
+
+ assert mock_init.called
+
+
+def test_mock_init_repository_update(mock_opam, tmp_path, datadir):
+ """Updating opam root directory with another instance should be ok
+
+ """
+ mock_init, mock_popen = mock_opam
+
+ instance = "fake_opam_repo"
+ instance_url = f"file://{datadir}/{instance}"
+ opam_root = str(tmp_path / "test-opam")
+
+ os.makedirs(opam_root, exist_ok=True)
+ with open(os.path.join(opam_root, "opam"), "w") as f:
+ f.write("one file to avoid empty folder")
+
+ assert os.path.exists(opam_root)
+ assert os.listdir(opam_root) == ["opam"] # not empty
+ # This will update the repository opam with another instance
+ opam_init(opam_root, instance, instance_url, {})
+
+ assert mock_init.called
+
+
def test_lister_opam_optional_instance(swh_scheduler):
"""Instance name should be optional and default to be built out of the netloc."""
netloc = "opam.ocaml.org"
@@ -36,21 +76,19 @@
lister = OpamLister(swh_scheduler, url=instance_url,)
assert lister.instance == netloc
- assert lister.opamroot.endswith(lister.instance)
+ assert lister.opam_root == "/tmp/opam/"
def test_urls(swh_scheduler, mock_opam):
mock_init, mock_popen = mock_opam
-
instance_url = "https://opam.ocaml.org"
+ tmp_folder = mkdtemp(prefix="swh_opam_lister")
lister = OpamLister(
- swh_scheduler,
- url=instance_url,
- instance="opam",
- opam_root=mkdtemp(prefix="swh_opam_lister"),
+ swh_scheduler, url=instance_url, instance="opam", opam_root=tmp_folder,
)
assert lister.instance == "opam"
+ assert lister.opam_root == tmp_folder
# call the lister and get all listed origins urls
stats = lister.run()
@@ -101,3 +139,32 @@
result_urls = [origin.url for origin in scheduler_origins]
assert expected_urls == result_urls
+
+
+def test_opam_multi_instance(datadir, swh_scheduler):
+ instance_url = f"file://{datadir}/fake_opam_repo"
+
+ lister = OpamLister(
+ swh_scheduler,
+ url=instance_url,
+ instance="fake",
+ opam_root=mkdtemp(prefix="swh_opam_lister"),
+ )
+
+ stats = lister.run()
+
+ assert stats.pages == 4
+ assert stats.origins == 4
+
+ scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
+
+ expected_urls = [
+ f"opam+{instance_url}/packages/agrid/",
+ f"opam+{instance_url}/packages/calculon/",
+ f"opam+{instance_url}/packages/directories/",
+ f"opam+{instance_url}/packages/ocb/",
+ ]
+
+ result_urls = [origin.url for origin in scheduler_origins]
+
+ assert expected_urls == result_urls