diff --git a/sql/upgrades/002.sql b/sql/upgrades/002.sql
new file mode 100644
--- /dev/null
+++ b/sql/upgrades/002.sql
@@ -0,0 +1,17 @@
+-- SWH DB schema upgrade
+-- from_version: 1
+-- to_version: 2
+-- description: add flag to acknowledge directories' flattening
+
+
+insert into dbversion(version, release, description)
+ values(2, now(), 'Work In Progress');
+
+alter table content
+ alter column date set not null;
+
+alter table directory
+ add column flat boolean not null default false;
+
+alter table directory
+ alter column date set not null;
diff --git a/swh/provenance/api/serializers.py b/swh/provenance/api/serializers.py
--- a/swh/provenance/api/serializers.py
+++ b/swh/provenance/api/serializers.py
@@ -33,11 +33,12 @@
ENCODERS: List[Tuple[type, str, Callable]] = [
+ (interface.EntityType, "enum", _encode_enum),
+ (interface.DirectoryData, "dataclass", _encode_dataclass),
(interface.ProvenanceResult, "dataclass", _encode_dataclass),
(interface.RelationData, "dataclass", _encode_dataclass),
- (interface.RevisionData, "dataclass", _encode_dataclass),
- (interface.EntityType, "enum", _encode_enum),
(interface.RelationType, "enum", _encode_enum),
+ (interface.RevisionData, "dataclass", _encode_dataclass),
(set, "set", list),
]
diff --git a/swh/provenance/api/server.py b/swh/provenance/api/server.py
--- a/swh/provenance/api/server.py
+++ b/swh/provenance/api/server.py
@@ -31,7 +31,13 @@
from swh.model.model import Sha1Git
from .. import get_provenance_storage
-from ..interface import EntityType, RelationData, RelationType, RevisionData
+from ..interface import (
+ DirectoryData,
+ EntityType,
+ RelationData,
+ RelationType,
+ RevisionData,
+)
from ..util import path_id
from .serializers import DECODERS, ENCODERS
@@ -53,21 +59,31 @@
pass
-def resolve_dates(
- dates: Iterable[Union[Tuple[Sha1Git, Optional[datetime]], Tuple[Sha1Git]]]
-) -> Dict[Sha1Git, Optional[datetime]]:
- result: Dict[Sha1Git, Optional[datetime]] = {}
- for row in dates:
- sha1 = row[0]
- date = (
- cast(Tuple[Sha1Git, Optional[datetime]], row)[1] if len(row) > 1 else None
- )
- known = result.setdefault(sha1, None)
- if date is not None and (known is None or date < known):
+def resolve_dates(dates: Iterable[Tuple[Sha1Git, datetime]]) -> Dict[Sha1Git, datetime]:
+ result: Dict[Sha1Git, datetime] = {}
+ for sha1, date in dates:
+ known = result.setdefault(sha1, date)
+ if date < known:
result[sha1] = date
return result
+def resolve_directory(
+ data: Iterable[Tuple[Sha1Git, DirectoryData]]
+) -> Dict[Sha1Git, DirectoryData]:
+ result: Dict[Sha1Git, DirectoryData] = {}
+ for sha1, dir in data:
+ known = result.setdefault(sha1, dir)
+ value = known
+ if dir.date < known.date:
+ value = DirectoryData(date=dir.date, flat=value.flat)
+ if dir.flat:
+ value = DirectoryData(date=value.date, flat=dir.flat)
+ if value != known:
+ result[sha1] = value
+ return result
+
+
def resolve_revision(
data: Iterable[Union[Tuple[Sha1Git, RevisionData], Tuple[Sha1Git]]]
) -> Dict[Sha1Git, RevisionData]:
@@ -458,8 +474,10 @@
@staticmethod
def get_conflicts_func(meth_name: str) -> Callable[[Iterable[Any]], Any]:
- if meth_name in ["content_add", "directory_add"]:
+ if meth_name == "content_add":
return resolve_dates
+ elif meth_name == "directory_add":
+ return resolve_directory
elif meth_name == "location_add":
return lambda data: set(data) # just remove duplicates
elif meth_name == "origin_add":
diff --git a/swh/provenance/interface.py b/swh/provenance/interface.py
--- a/swh/provenance/interface.py
+++ b/swh/provenance/interface.py
@@ -43,6 +43,18 @@
path: bytes
+@dataclass(eq=True, frozen=True)
+class DirectoryData:
+ """Object representing the data associated to a directory in the provenance model,
+ where `date` is the date of the directory in the isochrone frontier, and `flat` is a
+ flag acknowledging that a flat model for the elements outside the frontier has
+ already been created.
+ """
+
+ date: datetime
+ flat: bool
+
+
@dataclass(eq=True, frozen=True)
class RevisionData:
"""Object representing the data associated to a revision in the provenance model,
@@ -85,11 +97,9 @@
...
@remote_api_endpoint("content_add")
- def content_add(
- self, cnts: Union[Iterable[Sha1Git], Dict[Sha1Git, Optional[datetime]]]
- ) -> bool:
- """Add blobs identified by sha1 ids, with an optional associated date (as paired
- in `cnts`) to the provenance storage. Return a boolean stating whether the
+ def content_add(self, cnts: Dict[Sha1Git, datetime]) -> bool:
+ """Add blobs identified by sha1 ids, with an associated date (as paired in
+ `cnts`) to the provenance storage. Return a boolean stating whether the
information was successfully stored.
"""
...
@@ -108,25 +118,23 @@
@remote_api_endpoint("content_get")
def content_get(self, ids: Iterable[Sha1Git]) -> Dict[Sha1Git, datetime]:
- """Retrieve the associated date for each blob sha1 in `ids`. If some blob has
- no associated date, it is not present in the resulting dictionary.
- """
+ """Retrieve the associated date for each blob sha1 in `ids`."""
...
@remote_api_endpoint("directory_add")
- def directory_add(
- self, dirs: Union[Iterable[Sha1Git], Dict[Sha1Git, Optional[datetime]]]
- ) -> bool:
- """Add directories identified by sha1 ids, with an optional associated date (as
- paired in `dirs`) to the provenance storage. Return a boolean stating if the
- information was successfully stored.
+ def directory_add(self, dirs: Dict[Sha1Git, DirectoryData]) -> bool:
+ """Add directories identified by sha1 ids, with associated date and (optional)
+ flatten flag (as paired in `dirs`) to the provenance storage. If the flatten
+ flag is set to None, the previous value present in the storage is preserved.
+ Return a boolean stating if the information was successfully stored.
"""
...
@remote_api_endpoint("directory_get")
- def directory_get(self, ids: Iterable[Sha1Git]) -> Dict[Sha1Git, datetime]:
- """Retrieve the associated date for each directory sha1 in `ids`. If some
- directory has no associated date, it is not present in the resulting dictionary.
+ def directory_get(self, ids: Iterable[Sha1Git]) -> Dict[Sha1Git, DirectoryData]:
+ """Retrieve the associated date and (optional) flatten flag for each directory
+ sha1 in `ids`. If some directories has no associated date, it is not present in
+ the resulting dictionary.
"""
...
diff --git a/swh/provenance/mongo/backend.py b/swh/provenance/mongo/backend.py
--- a/swh/provenance/mongo/backend.py
+++ b/swh/provenance/mongo/backend.py
@@ -18,6 +18,7 @@
from swh.model.model import Sha1Git
from ..interface import (
+ DirectoryData,
EntityType,
ProvenanceResult,
ProvenanceStorageInterface,
@@ -52,21 +53,18 @@
self.db.client.close()
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "content_add"})
- def content_add(
- self, cnts: Union[Iterable[Sha1Git], Dict[Sha1Git, Optional[datetime]]]
- ) -> bool:
- data = cnts if isinstance(cnts, dict) else dict.fromkeys(cnts)
+ def content_add(self, cnts: Dict[Sha1Git, datetime]) -> bool:
existing = {
x["sha1"]: x
for x in self.db.content.find(
- {"sha1": {"$in": list(data)}}, {"sha1": 1, "ts": 1, "_id": 1}
+ {"sha1": {"$in": list(cnts)}}, {"sha1": 1, "ts": 1, "_id": 1}
)
}
- for sha1, date in data.items():
- ts = datetime.timestamp(date) if date is not None else None
+ for sha1, date in cnts.items():
+ ts = datetime.timestamp(date)
if sha1 in existing:
cnt = existing[sha1]
- if ts is not None and (cnt["ts"] is None or ts < cnt["ts"]):
+ if ts < cnt["ts"]:
self.db.content.update_one(
{"_id": cnt["_id"]}, {"$set": {"ts": ts}}
)
@@ -173,41 +171,43 @@
return {
x["sha1"]: datetime.fromtimestamp(x["ts"], timezone.utc)
for x in self.db.content.find(
- {"sha1": {"$in": list(ids)}, "ts": {"$ne": None}},
- {"sha1": 1, "ts": 1, "_id": 0},
+ {"sha1": {"$in": list(ids)}}, {"sha1": 1, "ts": 1, "_id": 0}
)
}
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "directory_add"})
- def directory_add(
- self, dirs: Union[Iterable[Sha1Git], Dict[Sha1Git, Optional[datetime]]]
- ) -> bool:
- data = dirs if isinstance(dirs, dict) else dict.fromkeys(dirs)
+ def directory_add(self, dirs: Dict[Sha1Git, DirectoryData]) -> bool:
existing = {
x["sha1"]: x
for x in self.db.directory.find(
- {"sha1": {"$in": list(data)}}, {"sha1": 1, "ts": 1, "_id": 1}
+ {"sha1": {"$in": list(dirs)}}, {"sha1": 1, "ts": 1, "flat": 1, "_id": 1}
)
}
- for sha1, date in data.items():
- ts = datetime.timestamp(date) if date is not None else None
+ for sha1, info in dirs.items():
+ ts = datetime.timestamp(info.date)
if sha1 in existing:
dir = existing[sha1]
- if ts is not None and (dir["ts"] is None or ts < dir["ts"]):
+ if ts >= dir["ts"]:
+ ts = dir["ts"]
+ flat = info.flat or dir["flat"]
+ if ts != dir["ts"] or flat != dir["flat"]:
self.db.directory.update_one(
- {"_id": dir["_id"]}, {"$set": {"ts": ts}}
+ {"_id": dir["_id"]}, {"$set": {"ts": ts, "flat": flat}}
)
else:
- self.db.directory.insert_one({"sha1": sha1, "ts": ts, "revision": {}})
+ self.db.directory.insert_one(
+ {"sha1": sha1, "ts": ts, "revision": {}, "flat": info.flat}
+ )
return True
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "directory_get"})
- def directory_get(self, ids: Iterable[Sha1Git]) -> Dict[Sha1Git, datetime]:
+ def directory_get(self, ids: Iterable[Sha1Git]) -> Dict[Sha1Git, DirectoryData]:
return {
- x["sha1"]: datetime.fromtimestamp(x["ts"], timezone.utc)
+ x["sha1"]: DirectoryData(
+ date=datetime.fromtimestamp(x["ts"], timezone.utc), flat=x["flat"]
+ )
for x in self.db.directory.find(
- {"sha1": {"$in": list(ids)}, "ts": {"$ne": None}},
- {"sha1": 1, "ts": 1, "_id": 0},
+ {"sha1": {"$in": list(ids)}}, {"sha1": 1, "ts": 1, "flat": 1, "_id": 0}
)
}
diff --git a/swh/provenance/postgresql/provenance.py b/swh/provenance/postgresql/provenance.py
--- a/swh/provenance/postgresql/provenance.py
+++ b/swh/provenance/postgresql/provenance.py
@@ -14,13 +14,13 @@
import psycopg2.extensions
import psycopg2.extras
-from typing_extensions import Literal
from swh.core.db import BaseDb
from swh.core.statsd import statsd
from swh.model.model import Sha1Git
from ..interface import (
+ DirectoryData,
EntityType,
ProvenanceResult,
ProvenanceStorageInterface,
@@ -82,10 +82,26 @@
self.conn.close()
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "content_add"})
- def content_add(
- self, cnts: Union[Iterable[Sha1Git], Dict[Sha1Git, Optional[datetime]]]
- ) -> bool:
- return self._entity_set_date("content", cnts)
+ def content_add(self, cnts: Dict[Sha1Git, datetime]) -> bool:
+ try:
+ if cnts:
+ sql = """
+ INSERT INTO content(sha1, date) VALUES %s
+ ON CONFLICT (sha1) DO
+ UPDATE SET date=LEAST(EXCLUDED.date,content.date)
+ """
+ page_size = self.page_size or len(cnts)
+ with self.transaction() as cursor:
+ psycopg2.extras.execute_values(
+ cursor, sql, argslist=cnts.items(), page_size=page_size
+ )
+ return True
+ except: # noqa: E722
+ # Unexpected error occurred, rollback all changes and log message
+ LOGGER.exception("Unexpected error")
+ if self.raise_on_commit:
+ raise
+ return False
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "content_find_first"})
def content_find_first(self, id: Sha1Git) -> Optional[ProvenanceResult]:
@@ -106,17 +122,67 @@
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "content_get"})
def content_get(self, ids: Iterable[Sha1Git]) -> Dict[Sha1Git, datetime]:
- return self._entity_get_date("content", ids)
+ dates: Dict[Sha1Git, datetime] = {}
+ sha1s = tuple(ids)
+ if sha1s:
+ # TODO: consider splitting this query in several ones if sha1s is too big!
+ values = ", ".join(itertools.repeat("%s", len(sha1s)))
+ sql = f"""
+ SELECT sha1, date
+ FROM content
+ WHERE sha1 IN ({values})
+ AND date IS NOT NULL
+ """
+ with self.transaction(readonly=True) as cursor:
+ cursor.execute(query=sql, vars=sha1s)
+ dates.update((row["sha1"], row["date"]) for row in cursor)
+ return dates
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "directory_add"})
- def directory_add(
- self, dirs: Union[Iterable[Sha1Git], Dict[Sha1Git, Optional[datetime]]]
- ) -> bool:
- return self._entity_set_date("directory", dirs)
+ def directory_add(self, dirs: Dict[Sha1Git, DirectoryData]) -> bool:
+ data = [(sha1, rev.date, rev.flat) for sha1, rev in dirs.items()]
+ try:
+ if data:
+ sql = """
+ INSERT INTO directory(sha1, date, flat) VALUES %s
+ ON CONFLICT (sha1) DO
+ UPDATE SET
+ date=LEAST(EXCLUDED.date, directory.date),
+ flat=(EXCLUDED.flat OR directory.flat)
+ """
+ page_size = self.page_size or len(data)
+ with self.transaction() as cursor:
+ psycopg2.extras.execute_values(
+ cur=cursor, sql=sql, argslist=data, page_size=page_size
+ )
+ return True
+ except: # noqa: E722
+ # Unexpected error occurred, rollback all changes and log message
+ LOGGER.exception("Unexpected error")
+ if self.raise_on_commit:
+ raise
+ return False
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "directory_get"})
- def directory_get(self, ids: Iterable[Sha1Git]) -> Dict[Sha1Git, datetime]:
- return self._entity_get_date("directory", ids)
+ def directory_get(self, ids: Iterable[Sha1Git]) -> Dict[Sha1Git, DirectoryData]:
+ result: Dict[Sha1Git, DirectoryData] = {}
+ sha1s = tuple(ids)
+ if sha1s:
+ # TODO: consider splitting this query in several ones if sha1s is too big!
+ values = ", ".join(itertools.repeat("%s", len(sha1s)))
+ sql = f"""
+ SELECT sha1, date, flat
+ FROM directory
+ WHERE sha1 IN ({values})
+ AND date IS NOT NULL
+ """
+ with self.transaction(readonly=True) as cursor:
+ cursor.execute(query=sql, vars=sha1s)
+ result.update(
+ (row["sha1"], DirectoryData(date=row["date"], flat=row["flat"]))
+ for row in cursor
+ )
+ return result
@statsd.timed(metric=STORAGE_DURATION_METRIC, tags={"method": "entity_get_all"})
def entity_get_all(self, entity: EntityType) -> Set[Sha1Git]:
@@ -299,53 +365,6 @@
) -> Dict[Sha1Git, Set[RelationData]]:
return self._relation_get(relation, None)
- def _entity_get_date(
- self,
- entity: Literal["content", "directory", "revision"],
- ids: Iterable[Sha1Git],
- ) -> Dict[Sha1Git, datetime]:
- dates: Dict[Sha1Git, datetime] = {}
- sha1s = tuple(ids)
- if sha1s:
- # TODO: consider splitting this query in several ones if sha1s is too big!
- values = ", ".join(itertools.repeat("%s", len(sha1s)))
- sql = f"""
- SELECT sha1, date
- FROM {entity}
- WHERE sha1 IN ({values})
- AND date IS NOT NULL
- """
- with self.transaction(readonly=True) as cursor:
- cursor.execute(query=sql, vars=sha1s)
- dates.update((row["sha1"], row["date"]) for row in cursor)
- return dates
-
- def _entity_set_date(
- self,
- entity: Literal["content", "directory"],
- dates: Union[Iterable[Sha1Git], Dict[Sha1Git, Optional[datetime]]],
- ) -> bool:
- data = dates if isinstance(dates, dict) else dict.fromkeys(dates)
- try:
- if data:
- sql = f"""
- INSERT INTO {entity}(sha1, date) VALUES %s
- ON CONFLICT (sha1) DO
- UPDATE SET date=LEAST(EXCLUDED.date,{entity}.date)
- """
- page_size = self.page_size or len(data)
- with self.transaction() as cursor:
- psycopg2.extras.execute_values(
- cursor, sql, argslist=data.items(), page_size=page_size
- )
- return True
- except: # noqa: E722
- # Unexpected error occurred, rollback all changes and log message
- LOGGER.exception("Unexpected error")
- if self.raise_on_commit:
- raise
- return False
-
def _relation_get(
self,
relation: RelationType,
diff --git a/swh/provenance/provenance.py b/swh/provenance/provenance.py
--- a/swh/provenance/provenance.py
+++ b/swh/provenance/provenance.py
@@ -15,6 +15,7 @@
from swh.model.model import Sha1Git
from .interface import (
+ DirectoryData,
ProvenanceInterface,
ProvenanceResult,
ProvenanceStorageInterface,
@@ -49,6 +50,7 @@
class ProvenanceCache(TypedDict):
content: DatetimeCache
directory: DatetimeCache
+ directory_flatten: Dict[Sha1Git, Optional[bool]]
revision: DatetimeCache
# below are insertion caches only
content_in_revision: Set[Tuple[Sha1Git, Sha1Git, bytes]]
@@ -65,6 +67,7 @@
return ProvenanceCache(
content=DatetimeCache(data={}, added=set()),
directory=DatetimeCache(data={}, added=set()),
+ directory_flatten={},
revision=DatetimeCache(data={}, added=set()),
content_in_revision=set(),
content_in_directory=set(),
@@ -194,7 +197,8 @@
# properly resolve any internal reference if needed. Content and directory
# entries may safely be registered with their associated dates. In contrast,
# revision entries should be registered without date, as it is used to
- # acknowledge that the flushing was successful.
+ # acknowledge that the flushing was successful. Also, directories are
+ # registered with their flatten flag not set.
cnt_dates = {
sha1: date
for sha1, date in self.cache["content"]["data"].items()
@@ -211,7 +215,7 @@
)
dir_dates = {
- sha1: date
+ sha1: DirectoryData(date=date, flat=False)
for sha1, date in self.cache["directory"]["data"].items()
if sha1 in self.cache["directory"]["added"] and date is not None
}
@@ -306,8 +310,27 @@
RelationType.DIR_IN_REV,
)
- # After relations, dates for the revisions can be safely set, acknowledging that
- # these revisions won't need to be reprocessed in case of failure.
+ # After relations, flatten flags for directories can be safely set (if
+ # applicable) acknowledging those directories that have already be flattened.
+ # Similarly, dates for the revisions are set to acknowledge that these revisions
+ # won't need to be reprocessed in case of failure.
+ dir_acks = {
+ sha1: DirectoryData(
+ date=date, flat=self.cache["directory_flatten"].get(sha1) or False
+ )
+ for sha1, date in self.cache["directory"]["data"].items()
+ if sha1 in self.cache["directory"]["added"] and date is not None
+ }
+ if dir_acks:
+ while not self.storage.directory_add(dir_acks):
+ statsd.increment(
+ metric=BACKEND_OPERATIONS_METRIC,
+ tags={"method": "flush_revision_content_retry_directory_ack"},
+ )
+ LOGGER.warning(
+ "Unable to write directory dates to the storage. Retrying..."
+ )
+
rev_dates = {
sha1: RevisionData(date=date, origin=None)
for sha1, date in self.cache["revision"]["data"].items()
@@ -388,14 +411,22 @@
cache = self.cache[entity]
missing_ids = set(id for id in ids if id not in cache)
if missing_ids:
- if entity == "revision":
- updated = {
- id: rev.date
- for id, rev in self.storage.revision_get(missing_ids).items()
- }
- else:
- updated = getattr(self.storage, f"{entity}_get")(missing_ids)
- cache["data"].update(updated)
+ if entity == "content":
+ cache["data"].update(self.storage.content_get(missing_ids))
+ elif entity == "directory":
+ cache["data"].update(
+ {
+ id: dir.date
+ for id, dir in self.storage.directory_get(missing_ids).items()
+ }
+ )
+ elif entity == "revision":
+ cache["data"].update(
+ {
+ id: rev.date
+ for id, rev in self.storage.revision_get(missing_ids).items()
+ }
+ )
dates: Dict[Sha1Git, datetime] = {}
for sha1 in ids:
date = cache["data"].setdefault(sha1, None)
diff --git a/swh/provenance/sql/30-schema.sql b/swh/provenance/sql/30-schema.sql
--- a/swh/provenance/sql/30-schema.sql
+++ b/swh/provenance/sql/30-schema.sql
@@ -16,7 +16,7 @@
-- latest schema version
insert into dbversion(version, release, description)
- values(1, now(), 'Work In Progress');
+ values(2, now(), 'Work In Progress');
-- a Git object ID, i.e., a Git-style salted SHA1 checksum
create domain sha1_git as bytea check (length(value) = 20);
@@ -35,9 +35,9 @@
-- entity tables
create table content
(
- id bigserial primary key, -- internal identifier of the content blob
- sha1 sha1_git unique not null, -- intrinsic identifier of the content blob
- date timestamptz -- timestamp of the revision where the blob appears early
+ id bigserial primary key, -- internal identifier of the content blob
+ sha1 sha1_git unique not null, -- intrinsic identifier of the content blob
+ date timestamptz not null -- timestamp of the revision where the blob appears early
);
comment on column content.id is 'Content internal identifier';
comment on column content.sha1 is 'Content intrinsic identifier';
@@ -45,9 +45,10 @@
create table directory
(
- id bigserial primary key, -- internal identifier of the directory appearing in an isochrone inner frontier
- sha1 sha1_git unique not null, -- intrinsic identifier of the directory
- date timestamptz -- max timestamp among those of the directory children's
+ id bigserial primary key, -- internal identifier of the directory appearing in an isochrone inner frontier
+ sha1 sha1_git unique not null, -- intrinsic identifier of the directory
+ date timestamptz not null, -- max timestamp among those of the directory children's
+ flat boolean not null default false -- flag acknowledging if the directory is flattenned in the model
);
comment on column directory.id is 'Directory internal identifier';
comment on column directory.sha1 is 'Directory intrinsic identifier';
@@ -55,10 +56,10 @@
create table revision
(
- id bigserial primary key, -- internal identifier of the revision
- sha1 sha1_git unique not null, -- intrinsic identifier of the revision
- date timestamptz, -- timestamp of the revision
- origin bigint -- id of the preferred origin
+ id bigserial primary key, -- internal identifier of the revision
+ sha1 sha1_git unique not null, -- intrinsic identifier of the revision
+ date timestamptz, -- timestamp of the revision
+ origin bigint -- id of the preferred origin
-- foreign key (origin) references origin (id)
);
comment on column revision.id is 'Revision internal identifier';
@@ -68,17 +69,17 @@
create table location
(
- id bigserial primary key, -- internal identifier of the location
- path unix_path unique not null -- path to the location
+ id bigserial primary key, -- internal identifier of the location
+ path unix_path unique not null -- path to the location
);
comment on column location.id is 'Location internal identifier';
comment on column location.path is 'Path to the location';
create table origin
(
- id bigserial primary key, -- internal identifier of the origin
- sha1 sha1_git unique not null, -- intrinsic identifier of the origin
- url text unique not null -- url of the origin
+ id bigserial primary key, -- internal identifier of the origin
+ sha1 sha1_git unique not null, -- intrinsic identifier of the origin
+ url text unique not null -- url of the origin
);
comment on column origin.id is 'Origin internal identifier';
comment on column origin.sha1 is 'Origin intrinsic identifier';
@@ -87,13 +88,13 @@
-- relation tables
create table content_in_revision
(
- content bigint not null, -- internal identifier of the content blob
+ content bigint not null, -- internal identifier of the content blob
\if :dbflavor_norm
- revision bigint not null, -- internal identifier of the revision where the blob appears for the first time
- location bigint -- location of the content relative to the revision's root directory
+ revision bigint not null, -- internal identifier of the revision where the blob appears for the first time
+ location bigint -- location of the content relative to the revision's root directory
\else
- revision bigint[], -- internal identifiers of the revisions where the blob appears for the first time
- location bigint[] -- locations of the content relative to the revisions' root directory
+ revision bigint[], -- internal identifiers of the revisions where the blob appears for the first time
+ location bigint[] -- locations of the content relative to the revisions' root directory
\endif
-- foreign key (content) references content (id),
-- foreign key (revision) references revision (id),
@@ -109,13 +110,13 @@
create table content_in_directory
(
- content bigint not null, -- internal identifier of the content blob
+ content bigint not null, -- internal identifier of the content blob
\if :dbflavor_norm
- directory bigint not null, -- internal identifier of the directory containing the blob
- location bigint -- location of the content relative to its parent directory in the isochrone frontier
+ directory bigint not null, -- internal identifier of the directory containing the blob
+ location bigint -- location of the content relative to its parent directory in the isochrone frontier
\else
- directory bigint[], -- internal reference of the directories containing the blob
- location bigint[] -- locations of the content relative to its parent directories in the isochrone frontier
+ directory bigint[], -- internal reference of the directories containing the blob
+ location bigint[] -- locations of the content relative to its parent directories in the isochrone frontier
\endif
-- foreign key (content) references content (id),
-- foreign key (directory) references directory (id),
@@ -131,13 +132,13 @@
create table directory_in_revision
(
- directory bigint not null, -- internal identifier of the directory appearing in the revision
+ directory bigint not null, -- internal identifier of the directory appearing in the revision
\if :dbflavor_norm
- revision bigint not null, -- internal identifier of the revision containing the directory
- location bigint -- location of the directory relative to the revision's root directory
+ revision bigint not null, -- internal identifier of the revision containing the directory
+ location bigint -- location of the directory relative to the revision's root directory
\else
- revision bigint[], -- internal identifiers of the revisions containing the directory
- location bigint[] -- locations of the directory relative to the revisions' root directory
+ revision bigint[], -- internal identifiers of the revisions containing the directory
+ location bigint[] -- locations of the directory relative to the revisions' root directory
\endif
-- foreign key (directory) references directory (id),
-- foreign key (revision) references revision (id),
@@ -153,8 +154,8 @@
create table revision_in_origin
(
- revision bigint not null, -- internal identifier of the revision poined by the origin
- origin bigint not null -- internal identifier of the origin that points to the revision
+ revision bigint not null, -- internal identifier of the revision poined by the origin
+ origin bigint not null -- internal identifier of the origin that points to the revision
-- foreign key (revision) references revision (id),
-- foreign key (origin) references origin (id)
);
@@ -163,8 +164,8 @@
create table revision_before_revision
(
- prev bigserial not null, -- internal identifier of the source revision
- next bigserial not null -- internal identifier of the destination revision
+ prev bigserial not null, -- internal identifier of the source revision
+ next bigserial not null -- internal identifier of the destination revision
-- foreign key (prev) references revision (id),
-- foreign key (next) references revision (id)
);
diff --git a/swh/provenance/tests/test_conflict_resolution.py b/swh/provenance/tests/test_conflict_resolution.py
--- a/swh/provenance/tests/test_conflict_resolution.py
+++ b/swh/provenance/tests/test_conflict_resolution.py
@@ -4,22 +4,29 @@
# See top-level LICENSE file for more information
from datetime import datetime
-from typing import List, Optional, Tuple, Union
+from typing import List, Tuple, Union
from swh.model.hashutil import hash_to_bytes
from swh.model.model import Sha1Git
-from swh.provenance.api.server import resolve_dates, resolve_relation, resolve_revision
-from swh.provenance.interface import RelationData, RevisionData
+from swh.provenance.api.server import (
+ resolve_dates,
+ resolve_directory,
+ resolve_relation,
+ resolve_revision,
+)
+from swh.provenance.interface import DirectoryData, RelationData, RevisionData
def test_resolve_dates() -> None:
- items: List[Union[Tuple[Sha1Git, Optional[datetime]], Tuple[Sha1Git]]] = [
- (hash_to_bytes("20329687bb9c1231a7e05afe86160343ad49b494"),),
+ items: List[Tuple[Sha1Git, datetime]] = [
+ (
+ hash_to_bytes("20329687bb9c1231a7e05afe86160343ad49b494"),
+ datetime.fromtimestamp(1000000001),
+ ),
(
hash_to_bytes("20329687bb9c1231a7e05afe86160343ad49b494"),
datetime.fromtimestamp(1000000000),
),
- (hash_to_bytes("20329687bb9c1231a7e05afe86160343ad49b494"), None),
]
assert resolve_dates(items) == {
hash_to_bytes(
@@ -28,21 +35,25 @@
}
-def test_resolve_dates_keep_min() -> None:
- items: List[Union[Tuple[Sha1Git, Optional[datetime]], Tuple[Sha1Git]]] = [
+def test_resolve_directory() -> None:
+ items: List[Tuple[Sha1Git, DirectoryData]] = [
(
- hash_to_bytes("20329687bb9c1231a7e05afe86160343ad49b494"),
- datetime.fromtimestamp(1000000001),
+ hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
+ DirectoryData(date=datetime.fromtimestamp(1000000002), flat=False),
),
(
- hash_to_bytes("20329687bb9c1231a7e05afe86160343ad49b494"),
- datetime.fromtimestamp(1000000000),
+ hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
+ DirectoryData(date=datetime.fromtimestamp(1000000001), flat=True),
+ ),
+ (
+ hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"),
+ DirectoryData(date=datetime.fromtimestamp(1000000000), flat=False),
),
]
- assert resolve_dates(items) == {
- hash_to_bytes(
- "20329687bb9c1231a7e05afe86160343ad49b494"
- ): datetime.fromtimestamp(1000000000)
+ assert resolve_directory(items) == {
+ hash_to_bytes("c0d8929936631ecbcf9147be6b8aa13b13b014e4"): DirectoryData(
+ date=datetime.fromtimestamp(1000000000), flat=True
+ )
}
diff --git a/swh/provenance/tests/test_provenance_storage.py b/swh/provenance/tests/test_provenance_storage.py
--- a/swh/provenance/tests/test_provenance_storage.py
+++ b/swh/provenance/tests/test_provenance_storage.py
@@ -12,6 +12,7 @@
from swh.model.model import Origin, Sha1Git
from swh.provenance.archive import ArchiveInterface
from swh.provenance.interface import (
+ DirectoryData,
EntityType,
ProvenanceInterface,
ProvenanceResult,
@@ -38,17 +39,12 @@
# Add all content present in the current repo to the storage, just assigning their
# creation dates. Then check that the returned results when querying are the same.
- cnts = {cnt["sha1_git"] for idx, cnt in enumerate(data["content"]) if idx % 2 == 0}
cnt_dates = {
- cnt["sha1_git"]: cnt["ctime"]
- for idx, cnt in enumerate(data["content"])
- if idx % 2 == 1
+ cnt["sha1_git"]: cnt["ctime"] for idx, cnt in enumerate(data["content"])
}
- assert cnts or cnt_dates
- assert provenance_storage.content_add(cnts)
assert provenance_storage.content_add(cnt_dates)
assert provenance_storage.content_get(set(cnt_dates.keys())) == cnt_dates
- assert provenance_storage.entity_get_all(EntityType.CONTENT) == cnts | set(
+ assert provenance_storage.entity_get_all(EntityType.CONTENT) == set(
cnt_dates.keys()
)
@@ -75,21 +71,15 @@
]
return max(dates) if dates else None
- dirs = {
- dir["id"]
- for dir in data["directory"]
- if getmaxdate(dir, data["content"]) is None
- }
- dir_dates = {
- dir["id"]: getmaxdate(dir, data["content"])
- for dir in data["directory"]
- if getmaxdate(dir, data["content"]) is not None
- }
- assert dirs
- assert provenance_storage.directory_add(dirs)
+ flat_values = (False, True)
+ dir_dates = {}
+ for idx, dir in enumerate(data["directory"]):
+ date = getmaxdate(dir, data["content"])
+ if date is not None:
+ dir_dates[dir["id"]] = DirectoryData(date=date, flat=flat_values[idx % 2])
assert provenance_storage.directory_add(dir_dates)
assert provenance_storage.directory_get(set(dir_dates.keys())) == dir_dates
- assert provenance_storage.entity_get_all(EntityType.DIRECTORY) == dirs | set(
+ assert provenance_storage.entity_get_all(EntityType.DIRECTORY) == set(
dir_dates.keys()
)
@@ -199,10 +189,13 @@
def entity_add(
storage: ProvenanceStorageInterface, entity: EntityType, ids: Set[Sha1Git]
) -> bool:
+ now = datetime.now(tz=timezone.utc)
if entity == EntityType.CONTENT:
- return storage.content_add({sha1: None for sha1 in ids})
+ return storage.content_add({sha1: now for sha1 in ids})
elif entity == EntityType.DIRECTORY:
- return storage.directory_add({sha1: None for sha1 in ids})
+ return storage.directory_add(
+ {sha1: DirectoryData(date=now, flat=False) for sha1 in ids}
+ )
else: # entity == EntityType.REVISION:
return storage.revision_add(
{sha1: RevisionData(date=None, origin=None) for sha1 in ids}
diff --git a/swh/provenance/tests/test_revision_content_layer.py b/swh/provenance/tests/test_revision_content_layer.py
--- a/swh/provenance/tests/test_revision_content_layer.py
+++ b/swh/provenance/tests/test_revision_content_layer.py
@@ -261,7 +261,9 @@
for rd in synth_rev["R_D"]:
assert (
rev_ts + rd["rel_ts"]
- == provenance.storage.directory_get([rd["dst"]])[rd["dst"]].timestamp()
+ == provenance.storage.directory_get([rd["dst"]])[
+ rd["dst"]
+ ].date.timestamp()
), synth_rev["msg"]
# ... + a number of rows in the "content_in_dir" table