Page MenuHomeSoftware Heritage
Differential D5036 Diff 17954 swh/clearlydefined/tests/test_orchestrator.py

Changeset View

Standalone View

View Options

swh/clearlydefined/tests/test_orchestrator.py

  • This file was added.
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.clearlydefined.orchestrator import orchestrator
from swh.clearlydefined.orchestrator import get_last_run_date
import psycopg2
from datetime import datetime
from typing import Optional
import gzip
import uuid
import os
from swh.model.model import Content
content_data = [
Content.from_data(b"42\n"),
Content.from_data(b"4242\n"),
]
def add_content_data(swh_storage):
swh_storage.content_add(content_data)
def file_data(file_name: str) -> str:
with open(file_name) as file:
data = file.read()
return data
vlorentzUnsubmitted
Done
Inline Actions
with open(file_name) as file:
- data = file.read()
- return data
+ return file.read()
def file_content(filename: Optional[str], datadir):
vlorentz:
def file_content(filename: Optional[str], datadir):
vlorentzUnsubmitted
Done
Inline Actions

better name + doc, please...

vlorentz: better name + doc, please...
if not filename:
return gzip.compress("".encode("utf-8"), compresslevel=9)
else:
return gzip.compress(
file_data(os.path.join(datadir, filename)).encode("utf-8"), compresslevel=9
)
def fill_rows_in_table(rows, cursor, connection):
vlorentzUnsubmitted
Done
Inline Actions

doc please (+ type annotation on rows wouldn't hurt)

vlorentz: doc please (+ type annotation on `rows` wouldn't hurt)
for row in rows:
cursor.execute(
"""INSERT INTO clearcode_cditem (path, content, last_modified_date,
last_map_date, map_error, uuid) VALUES (%s, %s, %s, %s, %s, %s);""",
(
row[0],
row[1],
row[2],
row[3],
row[4],
uuid.uuid4(),
),
vlorentzUnsubmitted
Done
Inline Actions
last_map_date, map_error, uuid) VALUES (%s, %s, %s, %s, %s, %s);""",
- (
- row[0],
- row[1],
- row[2],
- row[3],
- row[4],
- uuid.uuid4(),
- ),
+ (*row[0], uuid.uuid4()),
)
connection.commit()
vlorentz:
)
connection.commit()
def fill_data_before_updation_of_storage(connection, cursor, datadir):
rows = [
(
"maven/mavencentral/za.co.absa.cobrix/cobol-parser/revision/0.4.0.json",
file_content("definitions.json", datadir=datadir),
datetime(year=2021, month=2, day=1),
datetime(year=2021, month=2, day=1),
"",
),
(
"npm/npmjs/@ngtools/webpack/revision/10.2.1/tool/scancode/" "3.2.2.json",
file_content("scancode_true.json", datadir=datadir),
datetime(year=2021, month=2, day=2),
datetime(year=2021, month=2, day=2),
"",
),
(
"npm/npmjs/@fluidframework/replay-driver/revision/0.31.0/tool/licensee/"
"9.13.0.json",
file_content("licensee_true.json", datadir=datadir),
datetime(year=2021, month=2, day=3),
datetime(year=2021, month=2, day=3),
"",
),
(
"npm/npmjs/@pixi/mesh-extras/revision/5.3.5/tool/clearlydefined/1.3.4.json",
file_content("clearlydefined_true.json", datadir=datadir),
datetime(year=2021, month=2, day=4),
datetime(year=2021, month=2, day=4),
"",
),
(
"maven/mavencentral/za.co.absa.cobrix/cobol/revision/0.4.0.json",
file_content("def_not_mapped.json", datadir=datadir),
datetime(year=2021, month=2, day=5),
datetime(year=2021, month=2, day=5),
"",
),
(
"npm/npmjs/@pixi/mesh-extras/revision/5.3.6/tool/clearlydefined/1.3.4.json",
file_content("clearydefined_not_mapped.json", datadir=datadir),
datetime(year=2021, month=2, day=6),
datetime(year=2021, month=2, day=6),
"",
),
(
"npm/npmjs/@pixi/mesh-extras/revision/5.3.5/tool/fossology/1.3.4.json",
file_content(None, datadir=datadir),
datetime(year=2021, month=2, day=1),
datetime(year=2021, month=2, day=1),
"",
),
]
fill_rows_in_table(rows=rows, cursor=cursor, connection=connection)
def fill_data_after_updation_of_storage(connection, cursor, datadir):
rows = [
(
"maven/mavencentral/cobrix/cobol-parser/revision/0.4.0.json",
file_content(None, datadir=datadir),
datetime(year=2021, month=2, day=1),
datetime(year=2021, month=2, day=8),
"",
),
]
fill_rows_in_table(rows=rows, cursor=cursor, connection=connection)
def get_length_of_unmapped_data(connection, cursor) -> int:
cursor.execute("SELECT * FROM unmapped_data")
rows = cursor.fetchall()
return len(rows)
vlorentzUnsubmitted
Done
Inline Actions

There is a better way to do this in SQL

vlorentz: There is a better way to do this in SQL
def test_orchestrator(swh_storage, clearcode_dsn, datadir):
connection = psycopg2.connect(dsn=clearcode_dsn)
cursor = connection.cursor()
add_content_data(swh_storage)
# Fill data in clearcode database, for first time orchestration
fill_data_before_updation_of_storage(
connection=connection, cursor=cursor, datadir=datadir
)
orchestrator(storage=swh_storage, clearcode_dsn=clearcode_dsn)
# Check how much data is unmapped after first orchestration
assert 2 == get_length_of_unmapped_data(connection=connection, cursor=cursor)
assert datetime(
2021, 2, 6, 0, 0, tzinfo=psycopg2.tz.FixedOffsetTimezone(offset=0, name=None)
) == get_last_run_date(cursor=cursor)
vlorentzUnsubmitted
Done
Inline Actions
assert 2 == get_length_of_unmapped_data(connection=connection, cursor=cursor)
assert datetime(
- 2021, 2, 6, 0, 0, tzinfo=psycopg2.tz.FixedOffsetTimezone(offset=0, name=None)
+ 2021, 2, 6, 0, 0, tzinfo=datetime.timezone.utc
) == get_last_run_date(cursor=cursor)
content_data.extend(
vlorentz:
content_data.extend(
[Content.from_data(b"424242\n"), Content.from_data(b"42424242\n")]
)
add_content_data(swh_storage)
vlorentzUnsubmitted
Done
Inline Actions

This only works if the system's timezone is UTC

vlorentz: This only works if the system's timezone is UTC
TG1999AuthorUnsubmitted
Done
Inline Actions

Suggestion on this, what should be done?

TG1999: Suggestion on this, what should be done?
vlorentzUnsubmitted
Done
Inline Actions

Make get_last_run_date actually return a datetime instead of a string

vlorentz: Make `get_last_run_date` actually return a datetime instead of a string
# Run orchestration after updation in swh storage and
# check how much data is unmapped after second orchestration
vlorentzUnsubmitted
Done
Inline Actions
add_content_data(swh_storage)
- # Run orchestration after updation in swh storage and
+ # Run orchestration after insertion in swh storage and
# check how much data is unmapped after second orchestration
orchestrator(storage=swh_storage, clearcode_dsn=clearcode_dsn)
vlorentz:
orchestrator(storage=swh_storage, clearcode_dsn=clearcode_dsn)
assert 0 == get_length_of_unmapped_data(connection=connection, cursor=cursor)
fill_data_after_updation_of_storage(
connection=connection, cursor=cursor, datadir=datadir
)
# Fill new data in clearcode database and
# check how much data is unmapped after second orchestration
orchestrator(storage=swh_storage, clearcode_dsn=clearcode_dsn)
assert 1 == get_length_of_unmapped_data(connection=connection, cursor=cursor)
# Check how much data is unmapped when archive was not updated
orchestrator(storage=swh_storage, clearcode_dsn=clearcode_dsn)
assert 1 == get_length_of_unmapped_data(connection=connection, cursor=cursor)
About · Developer information · Legal