Differential D8225 Diff 29672 docker/tests/conftest.py

Changeset View

Standalone View

docker/tests/conftest.py

# Copyright (C) 2019-2021 The Software Heritage developers		# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from os.path import join		from os.path import join
import re		import re
import subprocess
import time		import time
from typing import Generator, Mapping, Tuple		from typing import Generator, Mapping, Tuple
from urllib.parse import urljoin		from urllib.parse import urljoin

import pytest		import pytest
import requests		import requests
import testinfra		import testinfra

Show All 14 Lines	<entry xmlns="http://www.w3.org/2005/Atom"
<swh:deposit>		<swh:deposit>
<swh:metadata-provenance>		<swh:metadata-provenance>
<schema:url>some-metadata-provenance-url</schema:url>		<schema:url>some-metadata-provenance-url</schema:url>
</swh:metadata-provenance>		</swh:metadata-provenance>
</swh:deposit>		</swh:deposit>
</entry>		</entry>
"""		"""

		# wait-for-it timout
		WFI_TIMEOUT = 60


		@pytest.fixture(scope="session")
		def docker_host():
		return testinfra.get_host("local://")


		@pytest.fixture(scope="session")
		def compose_cmd(docker_host):
		try:
		docker_host.check_output("docker compose version")
		return "docker compose"
		except AssertionError:
		print("Fall back to old docker-compose command")
		return "docker-compose"


# scope='session' so we use the same container for all the tests;		# scope='session' so we use the same container for all the tests;
@pytest.fixture(scope="session")		@pytest.fixture(scope="session")
def docker_compose(request):		def docker_compose(request, docker_host, compose_cmd):
# start the whole cluster		# start the whole cluster
subprocess.check_output(["docker-compose", "up", "-d"])		docker_host.check_output(f"{compose_cmd} up -d")
yield
# and stop it
subprocess.check_call(["docker-compose", "down", "-v"])

		# small hack: add a helper func to docker_host; so it's not necessary to
		# use all 3 docker_compose, docker_host and compose_cmd fixtures everywhere
		docker_host.check_compose_output = lambda command: docker_host.check_output(
		f"{compose_cmd} {command}"
		)
		yield docker_host

@pytest.fixture(scope="session")		# and stop the cluster
def wfi_timeout():		docker_host.check_output(f"{compose_cmd} down -v")
"""
wait-for-it timeout in seconds
"""
return 60


@pytest.fixture(scope="session")		@pytest.fixture(scope="session")
def scheduler_host(request, docker_compose, wfi_timeout):		def scheduler_host(request, docker_compose):
# run a container in which test commands are executed		# run a container in which test commands are executed
docker_id = (		docker_id = docker_compose.check_compose_output(
subprocess.check_output(		"run -d swh-scheduler shell sleep 1h"
["docker-compose", "run", "-d", "swh-scheduler", "shell", "sleep", "1h"]		).strip()
)
.decode()
.strip()
)
scheduler_host = testinfra.get_host("docker://" + docker_id)		scheduler_host = testinfra.get_host("docker://" + docker_id)
scheduler_host.check_output(f"wait-for-it swh-scheduler:5008 -t {wfi_timeout}")		scheduler_host.check_output(f"wait-for-it swh-scheduler:5008 -t {WFI_TIMEOUT}")
scheduler_host.check_output(f"wait-for-it swh-storage:5002 -t {wfi_timeout}")		scheduler_host.check_output(f"wait-for-it swh-storage:5002 -t {WFI_TIMEOUT}")

# return a testinfra connection to the container		# return a testinfra connection to the container
yield scheduler_host		yield scheduler_host

# at the end of the test suite, destroy the container		# at the end of the test suite, destroy the container
subprocess.check_call(["docker", "rm", "-f", docker_id])		docker_compose.check_output(f"docker rm -f {docker_id}")


# scope='session' so we use the same container for all the tests;		# scope='session' so we use the same container for all the tests;
@pytest.fixture(scope="session")		@pytest.fixture(scope="session")
def deposit_host(request, docker_compose, wfi_timeout):		def deposit_host(request, docker_compose):
# run a container in which test commands are executed		# run a container in which test commands are executed
docker_id = (		docker_id = docker_compose.check_compose_output(
subprocess.check_output(		"run -d swh-deposit shell sleep 1h"
["docker-compose", "run", "-d", "swh-deposit", "shell", "sleep", "1h"]		).strip()
)
.decode()
.strip()
)
deposit_host = testinfra.get_host("docker://" + docker_id)		deposit_host = testinfra.get_host("docker://" + docker_id)
deposit_host.check_output("echo 'print(\"Hello World!\")\n' > /tmp/hello.py")		deposit_host.check_output("echo 'print(\"Hello World!\")\n' > /tmp/hello.py")
deposit_host.check_output("tar -C /tmp -czf /tmp/archive.tgz /tmp/hello.py")		deposit_host.check_output("tar -C /tmp -czf /tmp/archive.tgz /tmp/hello.py")
deposit_host.check_output(f"echo '{SAMPLE_METADATA}' > /tmp/metadata.xml")		deposit_host.check_output(f"echo '{SAMPLE_METADATA}' > /tmp/metadata.xml")
deposit_host.check_output(f"wait-for-it swh-deposit:5006 -t {wfi_timeout}")		deposit_host.check_output(f"wait-for-it swh-deposit:5006 -t {WFI_TIMEOUT}")
# return a testinfra connection to the container		# return a testinfra connection to the container
yield deposit_host		yield deposit_host

# at the end of the test suite, destroy the container		# at the end of the test suite, destroy the container
subprocess.check_call(["docker", "rm", "-f", docker_id])		docker_compose.check_output(f"docker rm -f {docker_id}")


@pytest.fixture(scope="session")		@pytest.fixture(scope="session")
def git_url():		def git_url():
return "https://forge.softwareheritage.org/source/swh-core"		return "https://forge.softwareheritage.org/source/swh-core"


@pytest.fixture(scope="session")		@pytest.fixture(scope="session")
def git_origin(scheduler_host, git_url):		def git_origin(docker_compose, scheduler_host, git_url):
task = scheduler_host.check_output(f"swh scheduler task add load-git url={git_url}")		task = scheduler_host.check_output(f"swh scheduler task add load-git url={git_url}")
taskid = re.search(r"^Task (?P<id>\d+)$", task, flags=re.MULTILINE).group("id")		taskid = re.search(r"^Task (?P<id>\d+)$", task, flags=re.MULTILINE).group("id")
assert int(taskid) > 0		assert int(taskid) > 0

for i in range(60):		for i in range(60):
status = scheduler_host.check_output(		status = scheduler_host.check_output(
f"swh scheduler task list --list-runs --task-id {taskid}"		f"swh scheduler task list --list-runs --task-id {taskid}"
)		)
if "Executions:" in status:		if "Executions:" in status:
if "[eventful]" in status:		if "[eventful]" in status:
break		break
if "[started]" in status or "[scheduled]" in status:		if "[started]" in status or "[scheduled]" in status:
time.sleep(1)		time.sleep(1)
continue		continue
if "[failed]" in status:		if "[failed]" in status:
loader_logs = subprocess.check_output(		loader_logs = docker_compose.check_compose_output("logs swh-loader")
["docker-compose", "logs", "swh-loader"]
)
assert False, (		assert False, (
"Loading execution failed\n"		"Loading execution failed\n"
f"status: {status}\n"		f"status: {status}\n"
f"loader logs: " + loader_logs.decode(errors="replace")		f"loader logs: " + loader_logs
)		)
assert False, f"Loading execution failed, task status is {status}"		assert False, f"Loading execution failed, task status is {status}"
return git_url		return git_url


# Utility functions		# Utility functions


Show All 34 Lines