Page MenuHomeSoftware Heritage
Differential D5690

requirements-db-pytestplugin: Don't install pytest-postgresql 3.0+
ClosedPublic
Actions

Authored by vlorentz on May 5 2021, 7:51 PM.

Details

Reviewers
anlambert
Group Reviewers
Reviewers
Commits
rDCORE4d70ee0cfab6: requirements-db-pytestplugin: Don't install pytest-postgresql 3.0+
Summary

3.0 breaks backwards compatibility, we should handle the upgrade in a
separate patch, for now this ensures the rest of the pipelines can keep
running.

See also: D5688

Diff Detail

Repository
rDCORE Foundations and core functionalities
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

vlorentz created this revision.May 5 2021, 7:51 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptMay 5 2021, 7:51 PM
swh-public-ci added a comment.May 5 2021, 7:53 PM

Build is green

Patch application report for D5690 (id=20315)

Rebasing onto 7d42035a13...

Current branch diff-target is up to date.
Changes applied before test
commit 4d70ee0cfab62b79e20d60a8ac0b9351ed2f1963
Author: Valentin Lorentz <vlorentz@softwareheritage.org>
Date:   Wed May 5 19:50:46 2021 +0200

    requirements-db-pytestplugin: Don't install pytest-postgresql 3.0+
    
    3.0 breaks backwards compatibility, we should handle the upgrade in a
    separate patch, for now this ensures the rest of the pipelines can keep
    running.

See https://jenkins.softwareheritage.org/job/DCORE/job/tests-on-diff/220/ for more details.

Harbormaster completed remote builds in B21297: Diff 20315.May 5 2021, 7:53 PM
vlorentz requested review of this revision.May 5 2021, 7:53 PM
anlambert accepted this revision.May 5 2021, 8:37 PM
This revision is now accepted and ready to land.May 5 2021, 8:37 PM
Closed by commit rDCORE4d70ee0cfab6: requirements-db-pytestplugin: Don't install pytest-postgresql 3.0+ (authored by vlorentz). · Explain WhyMay 6 2021, 10:40 AM
This revision was automatically updated to reflect the committed changes.
vlorentz added a commit: rDCORE4d70ee0cfab6: requirements-db-pytestplugin: Don't install pytest-postgresql 3.0+.

Revision Contents
Changeset List

Diff 20329

View Options

requirements-db-pytestplugin.txt

# Copyright (C) 2020-2022 The Software Heritage developers # Copyright (C) 2020-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import collections import collections
from contextlib import contextmanager from contextlib import contextmanager
import math import math
from pathlib import Path from pathlib import Path
import tempfile import tempfile
import pyorc import pyorc
import pytest import pytest
from swh.dataset.exporters.orc import ( from swh.dataset.exporters import orc
ORCExporter,
SWHTimestampConverter,
datetime_to_tuple,
hash_to_hex_or_none,
swh_date_to_tuple,
)
from swh.dataset.relational import MAIN_TABLES, RELATION_TABLES from swh.dataset.relational import MAIN_TABLES, RELATION_TABLES
from swh.model.tests.swh_model_data import TEST_OBJECTS from swh.model.tests.swh_model_data import TEST_OBJECTS
from swh.objstorage.factory import get_objstorage
@contextmanager @contextmanager
def orc_tmpdir(tmpdir): def orc_tmpdir(tmpdir):
if tmpdir: if tmpdir:
yield Path(tmpdir) yield Path(tmpdir)
else: else:
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
yield Path(tmpdir) yield Path(tmpdir)
@contextmanager @contextmanager
def orc_export(messages, config=None, tmpdir=None): def orc_export(messages, config=None, tmpdir=None):
with orc_tmpdir(tmpdir) as tmpdir: with orc_tmpdir(tmpdir) as tmpdir:
if config is None: if config is None:
config = {} config = {}
with ORCExporter(config, tmpdir) as exporter: with orc.ORCExporter(config, tmpdir) as exporter:
for object_type, objects in messages.items(): for object_type, objects in messages.items():
for obj in objects: for obj in objects:
exporter.process_object(object_type, obj.to_dict()) exporter.process_object(object_type, obj.to_dict())
yield tmpdir yield tmpdir
def orc_load(rootdir): def orc_load(rootdir):
res = collections.defaultdict(list) res = collections.defaultdict(list)
res["rootdir"] = rootdir res["rootdir"] = rootdir
for obj_type_dir in rootdir.iterdir(): for obj_type_dir in rootdir.iterdir():
for orc_file in obj_type_dir.iterdir(): for orc_file in obj_type_dir.iterdir():
with orc_file.open("rb") as orc_obj: with orc_file.open("rb") as orc_obj:
reader = pyorc.Reader( reader = pyorc.Reader(
orc_obj, orc_obj,
converters={pyorc.TypeKind.TIMESTAMP: SWHTimestampConverter}, converters={pyorc.TypeKind.TIMESTAMP: orc.SWHTimestampConverter},
) )
obj_type = reader.user_metadata["swh_object_type"].decode() obj_type = reader.user_metadata["swh_object_type"].decode()
res[obj_type].extend(reader) res[obj_type].extend(reader)
return res return res
def exporter(messages, config=None, tmpdir=None): def exporter(messages, config=None, tmpdir=None):
with orc_export(messages, config, tmpdir) as exportdir: with orc_export(messages, config, tmpdir) as exportdir:
return orc_load(exportdir) return orc_load(exportdir)
def test_export_origin(): def test_export_origin():
obj_type = "origin" obj_type = "origin"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert (obj.url,) in output[obj_type] assert (obj.url,) in output[obj_type]
def test_export_origin_visit(): def test_export_origin_visit():
obj_type = "origin_visit" obj_type = "origin_visit"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert (obj.origin, obj.visit, datetime_to_tuple(obj.date), obj.type) in output[ assert (
obj_type obj.origin,
] obj.visit,
orc.datetime_to_tuple(obj.date),
obj.type,
) in output[obj_type]
def test_export_origin_visit_status(): def test_export_origin_visit_status():
obj_type = "origin_visit_status" obj_type = "origin_visit_status"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert ( assert (
obj.origin, obj.origin,
obj.visit, obj.visit,
datetime_to_tuple(obj.date), orc.datetime_to_tuple(obj.date),
obj.status, obj.status,
hash_to_hex_or_none(obj.snapshot), orc.hash_to_hex_or_none(obj.snapshot),
obj.type, obj.type,
) in output[obj_type] ) in output[obj_type]
def test_export_snapshot(): def test_export_snapshot():
obj_type = "snapshot" obj_type = "snapshot"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert (hash_to_hex_or_none(obj.id),) in output["snapshot"] assert (orc.hash_to_hex_or_none(obj.id),) in output["snapshot"]
for branch_name, branch in obj.branches.items(): for branch_name, branch in obj.branches.items():
if branch is None: if branch is None:
continue continue
assert ( assert (
hash_to_hex_or_none(obj.id), orc.hash_to_hex_or_none(obj.id),
branch_name, branch_name,
hash_to_hex_or_none(branch.target), orc.hash_to_hex_or_none(branch.target),
str(branch.target_type.value), str(branch.target_type.value),
) in output["snapshot_branch"] ) in output["snapshot_branch"]
def test_export_release(): def test_export_release():
obj_type = "release" obj_type = "release"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert ( assert (
hash_to_hex_or_none(obj.id), orc.hash_to_hex_or_none(obj.id),
obj.name, obj.name,
obj.message, obj.message,
hash_to_hex_or_none(obj.target), orc.hash_to_hex_or_none(obj.target),
obj.target_type.value, obj.target_type.value,
obj.author.fullname if obj.author else None, obj.author.fullname if obj.author else None,
*swh_date_to_tuple(obj.date.to_dict() if obj.date is not None else None), *orc.swh_date_to_tuple(
obj.date.to_dict() if obj.date is not None else None
),
obj.raw_manifest, obj.raw_manifest,
) in output[obj_type] ) in output[obj_type]
def test_export_revision(): def test_export_revision():
obj_type = "revision" obj_type = "revision"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert ( assert (
hash_to_hex_or_none(obj.id), orc.hash_to_hex_or_none(obj.id),
obj.message, obj.message,
obj.author.fullname, obj.author.fullname,
*swh_date_to_tuple(obj.date.to_dict() if obj.date is not None else None), *orc.swh_date_to_tuple(
obj.date.to_dict() if obj.date is not None else None
),
obj.committer.fullname, obj.committer.fullname,
*swh_date_to_tuple( *orc.swh_date_to_tuple(
obj.committer_date.to_dict() if obj.committer_date is not None else None obj.committer_date.to_dict() if obj.committer_date is not None else None
), ),
hash_to_hex_or_none(obj.directory), orc.hash_to_hex_or_none(obj.directory),
obj.type.value, obj.type.value,
obj.raw_manifest, obj.raw_manifest,
) in output["revision"] ) in output["revision"]
for i, parent in enumerate(obj.parents): for i, parent in enumerate(obj.parents):
assert ( assert (
hash_to_hex_or_none(obj.id), orc.hash_to_hex_or_none(obj.id),
hash_to_hex_or_none(parent), orc.hash_to_hex_or_none(parent),
i, i,
) in output["revision_history"] ) in output["revision_history"]
def test_export_directory(): def test_export_directory():
obj_type = "directory" obj_type = "directory"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert (hash_to_hex_or_none(obj.id), obj.raw_manifest) in output["directory"] assert (orc.hash_to_hex_or_none(obj.id), obj.raw_manifest) in output[
"directory"
]
for entry in obj.entries: for entry in obj.entries:
assert ( assert (
hash_to_hex_or_none(obj.id), orc.hash_to_hex_or_none(obj.id),
entry.name, entry.name,
entry.type, entry.type,
hash_to_hex_or_none(entry.target), orc.hash_to_hex_or_none(entry.target),
entry.perms, entry.perms,
) in output["directory_entry"] ) in output["directory_entry"]
def test_export_content(): def test_export_content():
obj_type = "content" obj_type = "content"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert ( assert (
hash_to_hex_or_none(obj.sha1), orc.hash_to_hex_or_none(obj.sha1),
hash_to_hex_or_none(obj.sha1_git), orc.hash_to_hex_or_none(obj.sha1_git),
hash_to_hex_or_none(obj.sha256), orc.hash_to_hex_or_none(obj.sha256),
hash_to_hex_or_none(obj.blake2s256), orc.hash_to_hex_or_none(obj.blake2s256),
obj.length, obj.length,
obj.status, obj.status,
None,
) in output[obj_type] ) in output[obj_type]
def test_export_skipped_content(): def test_export_skipped_content():
obj_type = "skipped_content" obj_type = "skipped_content"
output = exporter({obj_type: TEST_OBJECTS[obj_type]}) output = exporter({obj_type: TEST_OBJECTS[obj_type]})
for obj in TEST_OBJECTS[obj_type]: for obj in TEST_OBJECTS[obj_type]:
assert ( assert (
hash_to_hex_or_none(obj.sha1), orc.hash_to_hex_or_none(obj.sha1),
hash_to_hex_or_none(obj.sha1_git), orc.hash_to_hex_or_none(obj.sha1_git),
hash_to_hex_or_none(obj.sha256), orc.hash_to_hex_or_none(obj.sha256),
hash_to_hex_or_none(obj.blake2s256), orc.hash_to_hex_or_none(obj.blake2s256),
obj.length, obj.length,
obj.status, obj.status,
obj.reason, obj.reason,
) in output[obj_type] ) in output[obj_type]
def test_date_to_tuple(): def test_date_to_tuple():
ts = {"seconds": 123456, "microseconds": 1515} ts = {"seconds": 123456, "microseconds": 1515}
assert swh_date_to_tuple({"timestamp": ts, "offset_bytes": b"+0100"}) == ( assert orc.swh_date_to_tuple({"timestamp": ts, "offset_bytes": b"+0100"}) == (
(123456, 1515), (123456, 1515),
60, 60,
b"+0100", b"+0100",
) )
assert swh_date_to_tuple( assert orc.swh_date_to_tuple(
{ {
"timestamp": ts, "timestamp": ts,
"offset": 120, "offset": 120,
"negative_utc": False, "negative_utc": False,
"offset_bytes": b"+0100", "offset_bytes": b"+0100",
} }
) == ((123456, 1515), 60, b"+0100") ) == ((123456, 1515), 60, b"+0100")
assert swh_date_to_tuple( assert orc.swh_date_to_tuple(
{"timestamp": ts, "offset": 120, "negative_utc": False,} {"timestamp": ts, "offset": 120, "negative_utc": False,}
) == ((123456, 1515), 120, b"+0200") ) == ((123456, 1515), 120, b"+0200")
assert swh_date_to_tuple({"timestamp": ts, "offset": 0, "negative_utc": True,}) == ( assert orc.swh_date_to_tuple(
(123456, 1515), {"timestamp": ts, "offset": 0, "negative_utc": True,}
0, ) == ((123456, 1515), 0, b"-0000",)
b"-0000",
)
# mapping of related tables for each main table (if any) # mapping of related tables for each main table (if any)
RELATED = { RELATED = {
"snapshot": ["snapshot_branch"], "snapshot": ["snapshot_branch"],
"revision": ["revision_history", "revision_extra_headers"], "revision": ["revision_history", "revision_extra_headers"],
"directory": ["directory_entry"], "directory": ["directory_entry"],
} }
Show All 21 Lines for related in RELATED.get(obj_type, ()):
] ]
assert len(related_orcfiles) == len(orcfiles) assert len(related_orcfiles) == len(orcfiles)
# for each ORC file, check related files only reference objects in the # for each ORC file, check related files only reference objects in the
# corresponding main table # corresponding main table
for orc_file in orcfiles: for orc_file in orcfiles:
with orc_file.open("rb") as orc_obj: with orc_file.open("rb") as orc_obj:
reader = pyorc.Reader( reader = pyorc.Reader(
orc_obj, converters={pyorc.TypeKind.TIMESTAMP: SWHTimestampConverter}, orc_obj,
converters={pyorc.TypeKind.TIMESTAMP: orc.SWHTimestampConverter},
) )
uuid = reader.user_metadata["swh_uuid"].decode() uuid = reader.user_metadata["swh_uuid"].decode()
assert orc_file.basename == f"{obj_type}-{uuid}.orc" assert orc_file.basename == f"{obj_type}-{uuid}.orc"
rows = list(reader) rows = list(reader)
obj_ids = [row[0] for row in rows] obj_ids = [row[0] for row in rows]
# check the related tables # check the related tables
for related in RELATED.get(obj_type, ()): for related in RELATED.get(obj_type, ()):
orc_file = tmpdir / related / f"{related}-{uuid}.orc" orc_file = tmpdir / related / f"{related}-{uuid}.orc"
with orc_file.open("rb") as orc_obj: with orc_file.open("rb") as orc_obj:
reader = pyorc.Reader( reader = pyorc.Reader(
orc_obj, orc_obj,
converters={pyorc.TypeKind.TIMESTAMP: SWHTimestampConverter}, converters={pyorc.TypeKind.TIMESTAMP: orc.SWHTimestampConverter},
) )
assert reader.user_metadata["swh_uuid"].decode() == uuid assert reader.user_metadata["swh_uuid"].decode() == uuid
rows = list(reader) rows = list(reader)
# check branches in this file only concern current snapshot (obj_id) # check branches in this file only concern current snapshot (obj_id)
for row in rows: for row in rows:
assert row[0] in obj_ids assert row[0] in obj_ids
Show All 13 Lines for related in RELATED.get(obj_type, ()):
assert len(related_orcfiles) == len(orcfiles) assert len(related_orcfiles) == len(orcfiles)
@pytest.mark.parametrize("table_name", RELATION_TABLES.keys()) @pytest.mark.parametrize("table_name", RELATION_TABLES.keys())
def test_export_invalid_max_rows(table_name): def test_export_invalid_max_rows(table_name):
config = {"orc": {"max_rows": {table_name: 10}}} config = {"orc": {"max_rows": {table_name: 10}}}
with pytest.raises(ValueError): with pytest.raises(ValueError):
exporter({}, config=config) exporter({}, config=config)
def test_export_content_with_data(monkeypatch, tmpdir):
obj_type = "content"
objstorage = get_objstorage("memory")
for content in TEST_OBJECTS[obj_type]:
objstorage.add(content.data)
def get_objstorage_mock(**kw):
if kw.get("cls") == "mock":
return objstorage
monkeypatch.setattr(orc, "get_objstorage", get_objstorage_mock)
config = {
"orc": {"with_data": True, "objstorage": {"cls": "mock"},},
}
output = exporter({obj_type: TEST_OBJECTS[obj_type]}, config=config, tmpdir=tmpdir)
for obj in TEST_OBJECTS[obj_type]:
assert (
orc.hash_to_hex_or_none(obj.sha1),
orc.hash_to_hex_or_none(obj.sha1_git),
orc.hash_to_hex_or_none(obj.sha256),
orc.hash_to_hex_or_none(obj.blake2s256),
obj.length,
obj.status,
obj.data,
) in output[obj_type]
About · Developer information · Legal