Differential D5035 Diff 18042 swh/loader/package/nixguix/tests/test_nixguix.py

Changeset View

Standalone View

swh/loader/package/nixguix/tests/test_nixguix.py

# Copyright (C) 2020 The Software Heritage developers		# Copyright (C) 2020-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import json		import json
from json.decoder import JSONDecodeError
import logging		import logging
import os		import os
from typing import Dict, Optional, Tuple		from typing import Dict, Optional, Tuple
from unittest.mock import patch		from unittest.mock import patch

import attr		import attr
import pytest		import pytest

▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines


def test_retrieve_sources(swh_config, requests_mock_datadir):		def test_retrieve_sources(swh_config, requests_mock_datadir):
j = parse_sources(retrieve_sources(sources_url))		j = parse_sources(retrieve_sources(sources_url))
assert "sources" in j.keys()		assert "sources" in j.keys()
assert len(j["sources"]) == 2		assert len(j["sources"]) == 2


def test_retrieve_non_existing(swh_config, requests_mock_datadir):		def test_nixguix_url_not_found(swh_config, requests_mock_datadir):
with pytest.raises(ValueError):		"""When failing to read from the url, the visit is marked as not_found.
NixGuixLoader("https://non-existing-url")
		ardumontAuthorUnsubmitted Done Inline Actions What do the mocks do, exactly? Here the sources url is not found, so requests_mock_datadir returns a 404. Resulting in a NotFound within the base code from this diff. Which results in the task with status failed and a visit_status with status "not_found". ardumont: > What do the mocks do, exactly? Here the sources url is not found, so requests_mock_datadir…
		"""
		unknown_url = "https://non-existing-url"
		loader = NixGuixLoader(unknown_url)
		# during the retrieval step
		load_status = loader.load()

		assert load_status == {"status": "failed"}

		assert_last_visit_matches(
		loader.storage, unknown_url, status="not_found", type="nixguix", snapshot=None
		)


def test_retrieve_non_json(swh_config, requests_mock_datadir):		def test_nixguix_url_with_decoding_error(swh_config, requests_mock_datadir):
with pytest.raises(JSONDecodeError):		"""Other errors during communication with the url, the visit is marked as failed"""
NixGuixLoader("https://example.com/file.txt")		sources_url = "https://example.com/file.txt"
		loader = NixGuixLoader(sources_url)
		load_status = loader.load()

		assert load_status == {"status": "failed"}

		assert_last_visit_matches(
		loader.storage, sources_url, status="failed", type="nixguix", snapshot=None
		)

		vlorentzUnsubmitted Not Done Inline Actions what's going on here? What do the mocks do, exactly? vlorentz: what's going on here? What do the mocks do, exactly?
		ardumontAuthorUnsubmitted Done Inline Actions As before, the response returned a not json file which fails the json decode step. Thus triggering a JSONDecodeError caught by the main loop which ends up in the failed visit. That may be worth adding a docstring to explicit it. ardumont: As before, the response returned a not json file which fails the json decode step. Thus…
		ardumontAuthorUnsubmitted Done Inline Actions To clarify but that's the behavior for all tests really. What do the mocks do, exactly? requests_mock_datadir will intercept the requests to sources_url. Returns a 200 with the requested content of the query, here file.txt. As it's not a json file, that will fail do decode as explained in the prior comment. ardumont: To clarify but that's the behavior for all tests really. > What do the mocks do, exactly?
		vlorentzUnsubmitted Not Done Inline Actions could you add this in comments, so one doesn't need to jump to the datadir to understand what happens? vlorentz: could you add this in comments, so one doesn't need to jump to the datadir to understand what…
		ardumontAuthorUnsubmitted Done Inline Actions yes, i'll do it in another diff. ardumont: yes, i'll do it in another diff.

def test_clean_sources_invalid_schema(swh_config, requests_mock_datadir):		def test_clean_sources_invalid_schema(swh_config, requests_mock_datadir):
sources = {}		sources = {}
with pytest.raises(ValueError, match="sources structure invalid, missing: .*"):		with pytest.raises(ValueError, match="sources structure invalid, missing: .*"):
clean_sources(sources)		clean_sources(sources)


def test_clean_sources_invalid_version(swh_config, requests_mock_datadir):		def test_clean_sources_invalid_version(swh_config, requests_mock_datadir):
▲ Show 20 Lines • Show All 172 Lines • ▼ Show 20 Lines	def test_uncompress_failure(swh_config, requests_mock_datadir):
However, even in this case of failure (because of the url		However, even in this case of failure (because of the url
https://example.com/file.txt), a snapshot and a visit has to be		https://example.com/file.txt), a snapshot and a visit has to be
created (with a status partial since all files are not archived).		created (with a status partial since all files are not archived).

"""		"""
loader = NixGuixLoader(sources_url)		loader = NixGuixLoader(sources_url)
loader_status = loader.load()		loader_status = loader.load()

urls = [s["urls"][0] for s in loader.sources]		sources = loader.supported_sources()["sources"]
		urls = [s["urls"][0] for s in sources]
assert "https://example.com/file.txt" in urls		assert "https://example.com/file.txt" in urls
assert loader_status["status"] == "eventful"		assert loader_status["status"] == "eventful"

# The visit is partial because urls pointing to non tarball files		# The visit is partial because urls pointing to non tarball files
# are not handled yet		# are not handled yet
assert_last_visit_matches(		assert_last_visit_matches(
loader.storage, sources_url, status="partial", type="nixguix"		loader.storage, sources_url, status="partial", type="nixguix"
)		)
▲ Show 20 Lines • Show All 357 Lines • Show Last 20 Lines