diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,4 +1,4 @@ swh.core[db,http] >= 0.14.0 swh.counters >= v0.8.0 -swh.model >= 4.0.0 +swh.model >= 4.3.0 swh.objstorage >= 0.2.2 diff --git a/swh/storage/backfill.py b/swh/storage/backfill.py --- a/swh/storage/backfill.py +++ b/swh/storage/backfill.py @@ -115,12 +115,8 @@ "revision": [ ("revision.id", "id"), "date", - "date_offset", - "date_neg_utc_offset", "date_offset_bytes", "committer_date", - "committer_date_offset", - "committer_date_neg_utc_offset", "committer_date_offset_bytes", "type", "directory", @@ -146,8 +142,6 @@ "release": [ ("release.id", "id"), "date", - "date_offset", - "date_neg_utc_offset", "date_offset_bytes", "comment", ("release.name", "name"), diff --git a/swh/storage/cassandra/schema.py b/swh/storage/cassandra/schema.py --- a/swh/storage/cassandra/schema.py +++ b/swh/storage/cassandra/schema.py @@ -61,8 +61,6 @@ """ CREATE TYPE IF NOT EXISTS microtimestamp_with_timezone ( timestamp frozen, - offset smallint, - negative_utc boolean, offset_bytes blob, );""", """ diff --git a/swh/storage/postgresql/converters.py b/swh/storage/postgresql/converters.py --- a/swh/storage/postgresql/converters.py +++ b/swh/storage/postgresql/converters.py @@ -85,17 +85,14 @@ def db_to_date( - date: Optional[datetime.datetime], - offset: int, - neg_utc_offset: Optional[bool], - offset_bytes: Optional[bytes], + date: Optional[datetime.datetime], offset_bytes: bytes, ) -> Optional[TimestampWithTimezone]: """Convert the DB representation of a date to a swh-model compatible date. Args: date: a date pulled out of the database - offset: an integer number of minutes representing an UTC offset - neg_utc_offset: whether an utc offset is negative + offset_bytes: a byte representation of the latter two, usually as "+HHMM" + or "-HHMM" Returns: a TimestampWithTimezone, or None if the date is None. @@ -105,24 +102,13 @@ if date is None: return None - if neg_utc_offset is None: - # For older versions of the database that were not migrated to schema v160 - neg_utc_offset = False - - kwargs = {} - if offset_bytes: - # TODO: remove the conditional after migration is complete. - kwargs["offset_bytes"] = offset_bytes - return TimestampWithTimezone( timestamp=Timestamp( # we use floor() instead of int() to round down, because of negative dates seconds=math.floor(date.timestamp()), microseconds=date.microsecond, ), - offset=offset, - negative_utc=neg_utc_offset, - **kwargs, + offset_bytes=offset_bytes, ) @@ -136,9 +122,6 @@ dict: a dictionary with these keys: - timestamp: a date in ISO format - - offset: the UTC offset in minutes - - neg_utc_offset: a boolean indicating whether a null offset is - negative or positive. - offset_bytes: a byte representation of the latter two, usually as "+HHMM" or "-HHMM" @@ -156,7 +139,8 @@ # PostgreSQL supports isoformatted timestamps "timestamp": timestamp.isoformat(), "offset": ts_with_tz.offset, - "neg_utc_offset": ts_with_tz.negative_utc, + "neg_utc_offset": ts_with_tz.offset == 0 + and ts_with_tz.offset_bytes.startswith(b"-"), "offset_bytes": ts_with_tz.offset_bytes, } @@ -214,12 +198,7 @@ db_revision["author_name"], db_revision["author_email"], ) - date = db_to_date( - db_revision["date"], - db_revision["date_offset"], - db_revision["date_neg_utc_offset"], - db_revision["date_offset_bytes"], - ) + date = db_to_date(db_revision["date"], db_revision["date_offset_bytes"],) committer = db_to_author( db_revision["committer_fullname"], @@ -227,10 +206,7 @@ db_revision["committer_email"], ) committer_date = db_to_date( - db_revision["committer_date"], - db_revision["committer_date_offset"], - db_revision["committer_date_neg_utc_offset"], - db_revision["committer_date_offset_bytes"], + db_revision["committer_date"], db_revision["committer_date_offset_bytes"], ) assert author, "author is None" @@ -305,12 +281,7 @@ db_release["author_name"], db_release["author_email"], ) - date = db_to_date( - db_release["date"], - db_release["date_offset"], - db_release["date_neg_utc_offset"], - db_release["date_offset_bytes"], - ) + date = db_to_date(db_release["date"], db_release["date_offset_bytes"],) return Release( author=author, diff --git a/swh/storage/tests/algos/test_revisions_walker.py b/swh/storage/tests/algos/test_revisions_walker.py --- a/swh/storage/tests/algos/test_revisions_walker.py +++ b/swh/storage/tests/algos/test_revisions_walker.py @@ -100,13 +100,11 @@ "name": b"Adam", }, "committer_date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883849}, }, "date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883849}, }, "directory": b"\xefX\xe7\xa6\\\xda\xdf\xfdH\xdbH\xfbq\x96@{\x98?9\xfe", @@ -134,13 +132,11 @@ "name": b"Adam", }, "committer_date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883838}, }, "date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883838}, }, "directory": b"\xefX\xe7\xa6\\\xda\xdf\xfdH\xdbH\xfbq\x96@{\x98?9\xfe", @@ -167,13 +163,11 @@ "name": b"Adam", }, "committer_date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883838}, }, "date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883838}, }, "directory": b"\xefX\xe7\xa6\\\xda\xdf\xfdH\xdbH\xfbq\x96@{\x98?9\xfe", @@ -198,13 +192,11 @@ "name": b"Adam Janicki", }, "committer_date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883822}, }, "date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883822}, }, "directory": b"\xefX\xe7\xa6\\\xda\xdf\xfdH\xdbH\xfbq\x96@{\x98?9\xfe", @@ -229,13 +221,11 @@ "name": b"Adam", }, "committer_date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883692}, }, "date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883692}, }, "directory": b"d@\xe7\x143w\xcb\xf7\xad\xae\x91\xd5\xec\xd8\x95\x82" @@ -261,13 +251,11 @@ "name": b"Adam Janicki", }, "committer_date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883668}, }, "date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380883668}, }, "directory": b"\n\x857\x94r\xbe\xcc\x04=\xe9}\xe5\xfd\xdf?nR\xe6\xa7\x9e", @@ -292,13 +280,11 @@ "name": b"Adam Janicki", }, "committer_date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380881296}, }, "date": { - "negative_utc": None, - "offset": 120, + "offset_bytes": b"+0200", "timestamp": {"microseconds": 0, "seconds": 1380881296}, }, "directory": b".\xf9\xa5\xcb\xb0\xd3\xdc\x9b{\xb8\x81\x03l\xe2P\x16c\x0b|\xe6", # noqa @@ -323,13 +309,11 @@ "name": b"Adam", }, "committer_date": { - "negative_utc": None, - "offset": 60, + "offset_bytes": b"+0100", "timestamp": {"microseconds": 0, "seconds": 1380880096}, }, "date": { - "negative_utc": None, - "offset": 60, + "offset_bytes": b"+0100", "timestamp": {"microseconds": 0, "seconds": 1380880096}, }, "directory": b"\xc7r\xc4\x9f\xc0$\xd4\xab\xff\xcb]\xf6<\xcb\x8b~\xec\xc4\xd1)", # noqa @@ -354,13 +338,11 @@ "name": b"Adam", }, "committer_date": { - "negative_utc": None, - "offset": -420, + "offset_bytes": b"-0700", "timestamp": {"microseconds": 0, "seconds": 1380878419}, }, "date": { - "negative_utc": None, - "offset": -420, + "offset_bytes": b"-0700", "timestamp": {"microseconds": 0, "seconds": 1380878419}, }, "directory": b"WS\xbaX\xd6x{q\x8f\x020i\xc5\x95\xa01\xf7y\xb2\x80", diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_debian.py b/swh/storage/tests/migrate_extrinsic_metadata/test_debian.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_debian.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_debian.py @@ -214,13 +214,11 @@ committer=Person.from_fullname(b"foo"), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1580076204, microseconds=0), - offset=60, - negative_utc=False, + offset_bytes=b"+0100", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1580076204, microseconds=0), - offset=60, - negative_utc=False, + offset_bytes=b"+0100", ), type=RevisionType.DSC, directory=b"\xd5\x9a\x1f\x9c\x80\x9d\x8c}19P\xf6\xc8\xa2\x0f^%H\xcd\xdb", @@ -298,13 +296,11 @@ committer=Person.from_fullname(b"foo"), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1580076204, microseconds=0), - offset=60, - negative_utc=False, + offset_bytes=b"+0100", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1580076204, microseconds=0), - offset=60, - negative_utc=False, + offset_bytes=b"+0100", ), type=RevisionType.DSC, directory=b"\xd5\x9a\x1f\x9c\x80\x9d\x8c}19P\xf6\xc8\xa2\x0f^%H\xcd\xdb", diff --git a/swh/storage/tests/storage_data.py b/swh/storage/tests/storage_data.py --- a/swh/storage/tests/storage_data.py +++ b/swh/storage/tests/storage_data.py @@ -204,8 +204,7 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567890, microseconds=0), - offset=120, - negative_utc=False, + offset_bytes=b"+0200", ), committer=Person( name=b"St\xc3fano Zacchiroli", @@ -214,8 +213,7 @@ ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1123456789, microseconds=0), - offset=120, - negative_utc=False, + offset_bytes=b"+0200", ), parents=(), type=RevisionType.GIT, @@ -241,16 +239,14 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567843, microseconds=220000,), - offset=-720, - negative_utc=False, + offset_bytes=b"-1200", ), committer=Person( name=b"tony", email=b"ar@dumont.fr", fullname=b"tony ", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1123456789, microseconds=220000,), - offset=0, - negative_utc=False, + offset_bytes=b"+0000", ), parents=tuple([revision.id]), type=RevisionType.GIT, @@ -269,16 +265,14 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567843, microseconds=220000,), - offset=-720, - negative_utc=False, + offset_bytes=b"-1200", ), committer=Person( name=b"tony", email=b"ar@dumont.fr", fullname=b"tony ", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1127351742, microseconds=220000,), - offset=0, - negative_utc=False, + offset_bytes=b"+0000", ), parents=tuple([revision.id, revision2.id]), type=RevisionType.GIT, @@ -295,8 +289,7 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567843, microseconds=220000,), - offset=-720, - negative_utc=False, + offset_bytes=b"-1200", ), committer=Person( name=b"committer-dude", @@ -305,8 +298,7 @@ ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1244567843, microseconds=220000,), - offset=-720, - negative_utc=False, + offset_bytes=b"-1200", ), parents=tuple([revision3.id]), type=RevisionType.GIT, @@ -327,8 +319,7 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567890, microseconds=0), - offset=120, - negative_utc=False, + offset_bytes=b"+0200", ), committer=Person( name=b"St\xc3fano Zacchiroli", @@ -337,8 +328,7 @@ ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1123456789, microseconds=0), - offset=120, - negative_utc=False, + offset_bytes=b"+0200", ), parents=(), type=RevisionType.MERCURIAL, @@ -361,16 +351,14 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567843, microseconds=220000,), - offset=-720, - negative_utc=False, + offset_bytes=b"-1200", ), committer=Person( name=b"tony", email=b"ar@dumont.fr", fullname=b"tony ", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1123456789, microseconds=220000,), - offset=0, - negative_utc=False, + offset_bytes=b"+0000", ), parents=tuple([hg_revision.id]), type=RevisionType.MERCURIAL, @@ -391,16 +379,14 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567843, microseconds=220000,), - offset=-720, - negative_utc=False, + offset_bytes=b"-1200", ), committer=Person( name=b"tony", email=b"ar@dumont.fr", fullname=b"tony ", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1127351742, microseconds=220000,), - offset=0, - negative_utc=False, + offset_bytes=b"+0000", ), parents=tuple([hg_revision.id, hg_revision2.id]), type=RevisionType.MERCURIAL, @@ -419,8 +405,7 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567843, microseconds=220000,), - offset=-720, - negative_utc=False, + offset_bytes=b"-1200", ), committer=Person( name=b"committer-dude", @@ -429,8 +414,7 @@ ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1244567843, microseconds=220000,), - offset=-720, - negative_utc=False, + offset_bytes=b"-1200", ), parents=tuple([hg_revision3.id]), type=RevisionType.MERCURIAL, @@ -506,8 +490,7 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1234567890, microseconds=0), - offset=42, - negative_utc=False, + offset_bytes=b"+0042", ), target=revision.id, target_type=ObjectType.REVISION, @@ -522,8 +505,7 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1634366813, microseconds=0), - offset=-120, - negative_utc=False, + offset_bytes=b"-0200", ), target=revision2.id, target_type=ObjectType.REVISION, @@ -540,8 +522,7 @@ ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1634366813, microseconds=0), - offset=-120, - negative_utc=False, + offset_bytes=b"-0200", ), target=revision3.id, target_type=ObjectType.REVISION, diff --git a/swh/storage/tests/storage_tests.py b/swh/storage/tests/storage_tests.py --- a/swh/storage/tests/storage_tests.py +++ b/swh/storage/tests/storage_tests.py @@ -1088,8 +1088,7 @@ sample_data.revision, date=TimestampWithTimezone( timestamp=Timestamp(seconds=-1855958962, microseconds=0), - offset=0, - negative_utc=False, + offset_bytes=b"+0000", ), ) init_missing = swh_storage.revision_missing([revision.id]) diff --git a/swh/storage/tests/test_backfill.py b/swh/storage/tests/test_backfill.py --- a/swh/storage/tests/test_backfill.py +++ b/swh/storage/tests/test_backfill.py @@ -149,8 +149,6 @@ assert column_aliases == [ "id", "date", - "date_offset", - "date_neg_utc_offset", "date_offset_bytes", "comment", "name", @@ -167,7 +165,7 @@ assert ( query == """ -select release.id as id,date,date_offset,date_neg_utc_offset,date_offset_bytes,comment,release.name as name,synthetic,target,target_type,a.id as author_id,a.name as author_name,a.email as author_email,a.fullname as author_fullname,raw_manifest +select release.id as id,date,date_offset_bytes,comment,release.name as name,synthetic,target,target_type,a.id as author_id,a.name as author_name,a.email as author_email,a.fullname as author_fullname,raw_manifest from release left join person a on release.author=a.id where (release.id) >= %s and (release.id) < %s diff --git a/swh/storage/tests/test_postgresql_converters.py b/swh/storage/tests/test_postgresql_converters.py --- a/swh/storage/tests/test_postgresql_converters.py +++ b/swh/storage/tests/test_postgresql_converters.py @@ -35,8 +35,6 @@ ( TimestampWithTimezone( timestamp=Timestamp(seconds=1234567890, microseconds=0,), - offset=120, - negative_utc=False, offset_bytes=b"+0200", ), { @@ -49,8 +47,6 @@ ( TimestampWithTimezone( timestamp=Timestamp(seconds=1123456789, microseconds=0,), - offset=0, - negative_utc=True, offset_bytes=b"-0000", ), { @@ -63,8 +59,6 @@ ( TimestampWithTimezone( timestamp=Timestamp(seconds=1234567890, microseconds=0,), - offset=42, - negative_utc=False, offset_bytes=b"+0042", ), { @@ -77,8 +71,6 @@ ( TimestampWithTimezone( timestamp=Timestamp(seconds=1634366813, microseconds=0,), - offset=-120, - negative_utc=False, offset_bytes=b"-0200", ), { @@ -90,10 +82,7 @@ ), ( TimestampWithTimezone( - timestamp=Timestamp(seconds=0, microseconds=0,), - offset=-120, - negative_utc=False, - offset_bytes=b"-0200", + timestamp=Timestamp(seconds=0, microseconds=0,), offset_bytes=b"-0200", ), { "timestamp": "1970-01-01T00:00:00+00:00", @@ -104,10 +93,7 @@ ), ( TimestampWithTimezone( - timestamp=Timestamp(seconds=0, microseconds=1,), - offset=-120, - negative_utc=False, - offset_bytes=b"-0200", + timestamp=Timestamp(seconds=0, microseconds=1,), offset_bytes=b"-0200", ), { "timestamp": "1970-01-01T00:00:00.000001+00:00", @@ -118,10 +104,7 @@ ), ( TimestampWithTimezone( - timestamp=Timestamp(seconds=-1, microseconds=0,), - offset=-120, - negative_utc=False, - offset_bytes=b"-0200", + timestamp=Timestamp(seconds=-1, microseconds=0,), offset_bytes=b"-0200", ), { "timestamp": "1969-12-31T23:59:59+00:00", @@ -132,10 +115,7 @@ ), ( TimestampWithTimezone( - timestamp=Timestamp(seconds=-1, microseconds=1,), - offset=-120, - negative_utc=False, - offset_bytes=b"-0200", + timestamp=Timestamp(seconds=-1, microseconds=1,), offset_bytes=b"-0200", ), { "timestamp": "1969-12-31T23:59:59.000001+00:00", @@ -147,8 +127,6 @@ ( TimestampWithTimezone( timestamp=Timestamp(seconds=-3600, microseconds=0,), - offset=-120, - negative_utc=False, offset_bytes=b"-0200", ), { @@ -161,8 +139,6 @@ ( TimestampWithTimezone( timestamp=Timestamp(seconds=-3600, microseconds=1,), - offset=-120, - negative_utc=False, offset_bytes=b"-0200", ), { @@ -175,8 +151,6 @@ ( TimestampWithTimezone( timestamp=Timestamp(seconds=1234567890, microseconds=0,), - offset=120, - negative_utc=False, offset_bytes=b"+200", ), { @@ -195,8 +169,6 @@ date=None if db_date["timestamp"] is None else datetime.datetime.fromisoformat(db_date["timestamp"]), - offset=db_date["offset"], - neg_utc_offset=db_date["neg_utc_offset"], offset_bytes=db_date["offset_bytes"], ) == model_date