Page MenuHomeSoftware Heritage
Differential D3873

assets/origin-save: Fix handling of null visit dates in requests list
ClosedPublic
Actions

Authored by anlambert on Sep 3 2020, 3:17 PM.

Details

Reviewers
ardumont
Group Reviewers
Reviewers
Maniphest Tasks
T2557: "Save code now" history links lead to an error (because timestamp is null)
Commits
rDWAPPSf8a6cf8f2470: assets/origin-save: Fix handling of null visit dates in requests list
Summary

When a save request task succeeded but the corresponding visit is not yet available
in PostgreSQL database (replication lag), a null visit date can be returned so
handle that special case.

Closes T2557

Diff Detail

Repository
rDWAPPS Web applications
Branch
origin-save-ui-null-timestamp-fix
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 14898
Build 22958: Phabricator diff pipeline on jenkinsJenkins console · Jenkins
Build 22957: arc lint + arc unit

Event Timeline

anlambert created this revision.Sep 3 2020, 3:17 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptSep 3 2020, 3:17 PM
swh-public-ci added a comment.Sep 3 2020, 3:24 PM

Build is green

Patch application report for D3873 (id=13667)

Rebasing onto 9c41d9f90f...

Current branch diff-target is up to date.
Changes applied before test
commit 49c914f4d88cc45aeb1ae6f83717d37596ca0155
Author: Antoine Lambert <antoine.lambert@inria.fr>
Date:   Thu Sep 3 15:11:50 2020 +0200

    assets/origin-save: Fix handling of null visit dates in requests list
    
    Summary:
    
    When a save request task succeeded but the corresponding visit is not yet available
    in PostgreSQL database (replication lag), a null visit date can be returned so
    handle that special case.
    
    Closes T2557
    
    Test Plan:
    
    Reviewers:
    
    Subscribers:

See https://jenkins.softwareheritage.org/job/DWAPPS/job/tests-on-diff/351/ for more details.

Harbormaster completed remote builds in B14898: Diff 13667.Sep 3 2020, 3:24 PM
anlambert added a child revision: D3875: cypress/origin-save: Improve tests implementation.Sep 3 2020, 3:27 PM
ardumont accepted this revision.Sep 3 2020, 6:01 PM
This revision is now accepted and ready to land.Sep 3 2020, 6:01 PM
Closed by commit rDWAPPSf8a6cf8f2470: assets/origin-save: Fix handling of null visit dates in requests list (authored by anlambert). · Explain WhySep 4 2020, 2:04 PM
This revision was automatically updated to reflect the committed changes.
anlambert added a commit: rDWAPPSf8a6cf8f2470: assets/origin-save: Fix handling of null visit dates in requests list.

Revision Contents
Changeset List

PathSizeCoverage (All)Coverage (Touched)
cypress/
integration/
23 lines--
swh/
web/
assets/
src/
bundles/
admin/
4 lines54%
Loading...
save/
4 lines82%
Loading...

Diff 13667

View Options

cypress/integration/origin-save.spec.js

  • This file was added.
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from datetime import datetime
import pytest
from swh.model.hashutil import hash_to_bytes
from swh.provenance.api.client import split_ranges
from swh.provenance.interface import RelationData, RelationType
def test_split_ranges_for_relation() -> None:
data = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/1"
),
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/2"
),
},
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/3"
),
},
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/4"
),
},
}
assert split_ranges(data, "relation_add", RelationType.CNT_EARLY_IN_REV) == {
"relation_add.content_in_revision.c": {
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/1",
),
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/2",
),
(
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/4",
),
},
"relation_add.content_in_revision.d": {
(
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/3",
),
},
}
def test_split_ranges_error_for_relation() -> None:
set_data = {hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4")}
with pytest.raises(BaseException) as ex:
vlorentzUnsubmitted
Not Done
Inline Actions

Same comment as the other diff

vlorentz: Same comment as the other diff
split_ranges(set_data, "relation_add", RelationType.CNT_EARLY_IN_REV)
assert "Relation data must be provided in a dictionary" in str(ex.value)
tuple_values = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"): (
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/3",
)
}
with pytest.raises(BaseException) as ex:
vlorentzUnsubmitted
Not Done
Inline Actions

ditto

vlorentz: ditto
split_ranges(tuple_values, "relation_add", RelationType.CNT_EARLY_IN_REV)
assert "Values in the dictionary must be RelationData structures" in str(ex.value)
@pytest.mark.parametrize(
"entity",
("content", "directory", "origin", "revision"),
)
def test_split_ranges_for_entity_without_data(entity: str) -> None:
data = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
}
meth_name = f"{entity}_add"
assert split_ranges(data, meth_name, None) == {
f"{meth_name}.unknown.c": {
(hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),),
(hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),),
},
f"{meth_name}.unknown.d": {
(hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),),
},
}
@pytest.mark.parametrize(
"entity",
("content", "directory", "origin", "revision"),
)
def test_split_ranges_for_entity_with_data(entity: str) -> None:
data = {
hash_to_bytes(
"c0d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000000),
hash_to_bytes(
"d0d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000001),
hash_to_bytes(
"c1d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000002),
}
meth_name = f"{entity}_add"
assert split_ranges(data, meth_name, None) == {
f"{meth_name}.unknown.c": {
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000000),
),
(
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000002),
),
},
f"{meth_name}.unknown.d": {
(
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000001),
),
},
}
View Options

swh/web/assets/src/bundles/admin/origin-save.js

  • This file was added.
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from datetime import datetime
import pytest
from swh.model.hashutil import hash_to_bytes
from swh.provenance.api.client import split_ranges
from swh.provenance.interface import RelationData, RelationType
def test_split_ranges_for_relation() -> None:
data = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/1"
),
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/2"
),
},
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/3"
),
},
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/4"
),
},
}
assert split_ranges(data, "relation_add", RelationType.CNT_EARLY_IN_REV) == {
"relation_add.content_in_revision.c": {
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/1",
),
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/2",
),
(
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/4",
),
},
"relation_add.content_in_revision.d": {
(
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/3",
),
},
}
def test_split_ranges_error_for_relation() -> None:
set_data = {hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4")}
with pytest.raises(BaseException) as ex:
vlorentzUnsubmitted
Not Done
Inline Actions

Same comment as the other diff

vlorentz: Same comment as the other diff
split_ranges(set_data, "relation_add", RelationType.CNT_EARLY_IN_REV)
assert "Relation data must be provided in a dictionary" in str(ex.value)
tuple_values = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"): (
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/3",
)
}
with pytest.raises(BaseException) as ex:
vlorentzUnsubmitted
Not Done
Inline Actions

ditto

vlorentz: ditto
split_ranges(tuple_values, "relation_add", RelationType.CNT_EARLY_IN_REV)
assert "Values in the dictionary must be RelationData structures" in str(ex.value)
@pytest.mark.parametrize(
"entity",
("content", "directory", "origin", "revision"),
)
def test_split_ranges_for_entity_without_data(entity: str) -> None:
data = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
}
meth_name = f"{entity}_add"
assert split_ranges(data, meth_name, None) == {
f"{meth_name}.unknown.c": {
(hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),),
(hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),),
},
f"{meth_name}.unknown.d": {
(hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),),
},
}
@pytest.mark.parametrize(
"entity",
("content", "directory", "origin", "revision"),
)
def test_split_ranges_for_entity_with_data(entity: str) -> None:
data = {
hash_to_bytes(
"c0d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000000),
hash_to_bytes(
"d0d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000001),
hash_to_bytes(
"c1d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000002),
}
meth_name = f"{entity}_add"
assert split_ranges(data, meth_name, None) == {
f"{meth_name}.unknown.c": {
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000000),
),
(
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000002),
),
},
f"{meth_name}.unknown.d": {
(
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000001),
),
},
}
View Options

swh/web/assets/src/bundles/save/index.js

  • This file was added.
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from datetime import datetime
import pytest
from swh.model.hashutil import hash_to_bytes
from swh.provenance.api.client import split_ranges
from swh.provenance.interface import RelationData, RelationType
def test_split_ranges_for_relation() -> None:
data = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/1"
),
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/2"
),
},
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/3"
),
},
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"): {
RelationData(
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"), b"/path/4"
),
},
}
assert split_ranges(data, "relation_add", RelationType.CNT_EARLY_IN_REV) == {
"relation_add.content_in_revision.c": {
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/1",
),
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/2",
),
(
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/4",
),
},
"relation_add.content_in_revision.d": {
(
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/3",
),
},
}
def test_split_ranges_error_for_relation() -> None:
set_data = {hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4")}
with pytest.raises(BaseException) as ex:
vlorentzUnsubmitted
Not Done
Inline Actions

Same comment as the other diff

vlorentz: Same comment as the other diff
split_ranges(set_data, "relation_add", RelationType.CNT_EARLY_IN_REV)
assert "Relation data must be provided in a dictionary" in str(ex.value)
tuple_values = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"): (
hash_to_bytes("3acef14580ea7fd42840ee905c5ce2b0ef9e8174"),
b"/path/3",
)
}
with pytest.raises(BaseException) as ex:
vlorentzUnsubmitted
Not Done
Inline Actions

ditto

vlorentz: ditto
split_ranges(tuple_values, "relation_add", RelationType.CNT_EARLY_IN_REV)
assert "Values in the dictionary must be RelationData structures" in str(ex.value)
@pytest.mark.parametrize(
"entity",
("content", "directory", "origin", "revision"),
)
def test_split_ranges_for_entity_without_data(entity: str) -> None:
data = {
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
}
meth_name = f"{entity}_add"
assert split_ranges(data, meth_name, None) == {
f"{meth_name}.unknown.c": {
(hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),),
(hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),),
},
f"{meth_name}.unknown.d": {
(hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),),
},
}
@pytest.mark.parametrize(
"entity",
("content", "directory", "origin", "revision"),
)
def test_split_ranges_for_entity_with_data(entity: str) -> None:
data = {
hash_to_bytes(
"c0d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000000),
hash_to_bytes(
"d0d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000001),
hash_to_bytes(
"c1d8929936631ecbcf9147be6b8aa13b13b014e4"
): datetime.fromtimestamp(1000000002),
}
meth_name = f"{entity}_add"
assert split_ranges(data, meth_name, None) == {
f"{meth_name}.unknown.c": {
(
hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000000),
),
(
hash_to_bytes("c1d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000002),
),
},
f"{meth_name}.unknown.d": {
(
hash_to_bytes("d0d8929936631ecbcf9147be6b8aa13b13b014e4"),
datetime.fromtimestamp(1000000001),
),
},
}
About · Developer information · Legal