View Options

	diff --git a/docker/tests/test_git_loader.py b/docker/tests/test_git_loader.py
	index b8c2d43..0475fc3 100644
	--- a/docker/tests/test_git_loader.py
	+++ b/docker/tests/test_git_loader.py
	@@ -1,61 +1,79 @@
	# Copyright (C) 2019-2021 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	from urllib.parse import quote_plus

	+from dulwich import porcelain
	+from dulwich.repo import MemoryRepo
	+
	from .conftest import apiget


	def test_git_loader(scheduler_host, git_origin):
	url = git_origin

	print(f"Retrieve references available at {url}")
	- gitrefs = scheduler_host.check_output(f"git ls-remote {url}")
	- gitrefs = [x.split() for x in gitrefs.splitlines()]
	+ repo = MemoryRepo()
	+ gitrefs = porcelain.fetch(repo, url).refs

	print(f"Look for origin {url}")
	# use quote_plus to prevent urljoin from messing with the 'http://' part of
	# the url
	origin = apiget(f"origin/{quote_plus(url)}/get")
	assert origin["url"] == url

	visit = apiget(f"origin/{quote_plus(url)}/visit/latest")
	assert visit["status"] == "full"

	print("Check every identified git ref has been loaded")
	snapshot = apiget(f'snapshot/{visit["snapshot"]}')

	print(f'snapshot has {len(snapshot["branches"])} branches')
	branches = snapshot["branches"]

	- # check every branch reported by git ls-remote is present in the snapshot
	- for rev, branch_name in gitrefs:
	+ # check every fetched branch is present in the snapshot
	+ for branch_name, rev in gitrefs.items():
	# for tags, only check for final revision id
	- if branch_name.startswith("refs/tags/") and not branch_name.endswith("^{}"):
	+ if branch_name.startswith(b"refs/tags/") and not branch_name.endswith(b"^{}"):
	continue
	- rev_desc = apiget(f"revision/{rev}")
	+ rev_desc = apiget(f"revision/{rev.decode()}")
	assert rev_desc["type"] == "git"

	tag_revision = {}
	tag_release = {}
	- for rev, tag in gitrefs:
	- if tag.startswith("refs/tags/"):
	- if tag.endswith("^{}"):
	- tag_revision[tag[:-3]] = rev
	+ for tag, rev in gitrefs.items():
	+ if tag.startswith(b"refs/tags/"):
	+ tag_str = tag.decode()
	+ rev_str = rev.decode()
	+ if tag.endswith(b"^{}"):
	+ tag_revision[tag_str[:-3]] = rev_str
	else:
	- tag_release[tag] = rev
	+ tag_release[tag_str] = rev_str

	for tag, revision in tag_revision.items():
	# check that every release tag listed in the snapshot is known by the
	# archive and consistent
	release_id = tag_release[tag]
	release = apiget(f"release/{release_id}")
	assert release["id"] == release_id
	assert release["target_type"] == "revision"
	assert release["target"] == revision
	# and compare this with what git ls-remote reported
	tag_desc = branches[tag]
	assert tag_desc["target_type"] == "release"
	assert tag_desc["target"] == release_id
	+
	+ print("Check every git object stored in the repository has been loaded")
	+ for sha1 in repo.object_store:
	+ obj = repo.get_object(sha1)
	+ sha1_str = sha1.decode()
	+ if obj.type_name == b"blob":
	+ apiget(f"content/sha1_git:{sha1_str}")
	+ elif obj.type_name == b"commit":
	+ apiget(f"revision/{sha1_str}")
	+ elif obj.type_name == b"tree":
	+ apiget(f"directory/{sha1_str}")
	+ elif obj.type_name == b"tag":
	+ apiget(f"release/{sha1_str}")
	diff --git a/docker/tox.ini b/docker/tox.ini
	index 54565f4..bae5e16 100644
	--- a/docker/tox.ini
	+++ b/docker/tox.ini
	@@ -1,38 +1,39 @@
	[tox]
	envlist=flake8,py3,shell_tests

	[testenv]
	skip_install = true
	passenv = COMPOSE_FILE
	deps =
	pytest
	pytest-testinfra
	docker-compose
	pdbpp
	+ dulwich
	commands =
	## this 'build' step is disabled for now because it fails with a somewhat
	## cryptic traceback about the secretservice library. So it must be managed
	## by hand from outside the tox invocation.
	# docker-compose build swh-storage
	pytest {posargs}

	[testenv:flake8]
	deps =
	flake8
	commands =
	{envpython} -m flake8 tests

	[testenv:shell_tests]
	deps =
	docker-compose
	whitelist_externals =
	/bin/bash
	commands =
	/bin/bash tests/run_tests.sh

	[flake8]
	# E203: whitespaces before ':' <https://github.com/psf/black/issues/315>
	# E231: missing whitespace after ','
	# W503: line break before binary operator <https://github.com/psf/black/issues/52>
	ignore = E203,E231,W503
	max-line-length = 88

File Metadata

Mime Type: text/x-diff
Expires: Fri, Jul 4, 12:16 PM (2 w, 3 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3334532

No OneTemporary
Actions

View Options

File Metadata

Event Timeline

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions