swh_storage_backend_config = {'cls': 'local', 'db': 'postgresql://postgres@127.0.0.1:29727/tests', 'journal_writer': {'brokers': ['127.0.0.1:52229'], 'client_id': 'kafka_writer-1', 'cls': 'kafka', 'prefix': 'ghwzsjgfkj-1'}, 'objstorage': {'args': {}, 'cls': 'memory'}} kafka_prefix = 'ghwzsjgfkj', kafka_consumer_group = 'test-consumer-ghwzsjgfkj' kafka_server = '127.0.0.1:52229' @patch("swh.storage.backfill.RANGE_GENERATORS", RANGE_GENERATORS) def test_backfiller( swh_storage_backend_config, kafka_prefix: str, kafka_consumer_group: str, kafka_server: str, ): prefix1 = f"{kafka_prefix}-1" prefix2 = f"{kafka_prefix}-2" journal1 = { "cls": "kafka", "brokers": [kafka_server], "client_id": "kafka_writer-1", "prefix": prefix1, } swh_storage_backend_config["journal_writer"] = journal1 storage = get_storage(**swh_storage_backend_config) # fill the storage and the journal (under prefix1) for object_type, objects in TEST_OBJECTS.items(): method = getattr(storage, object_type + "_add") > method(objects) .tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_backfill.py:204: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .tox/py3/lib/python3.7/site-packages/swh/storage/metrics.py:24: in d return f(*a, **kw) .tox/py3/lib/python3.7/site-packages/swh/core/db/common.py:62: in _meth return meth(self, *args, db=db, cur=cur, **kwargs) .tox/py3/lib/python3.7/site-packages/swh/storage/storage.py:821: in origin_visit_add db.origin_visit_upsert(visit) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <swh.storage.db.Db object at 0x7f743fc210f0> origin_visit = OriginVisit(origin='https://somewhere.org/den/fox', date=datetime.datetime(2013, 5, 7, 4, 20, 39, 369271, tzinfo=datetime.timezone.utc), type='git', visit=1, status=None, snapshot=None, metadata=None) cur = <cursor object at 0x7f743f2b19f8; closed: 0> def origin_visit_upsert(self, origin_visit: OriginVisit, cur=None) -> None: # doing an extra query like this is way simpler than trying to join # the origin id in the query below ov = origin_visit origin_id = next(self.origin_id_get_by_url([ov.origin])) cur = self._cursor(cur) query = """INSERT INTO origin_visit ({cols}) VALUES ({values}) ON CONFLICT ON CONSTRAINT origin_visit_pkey DO UPDATE SET {updates}""".format( cols=", ".join(self.origin_visit_get_cols), values=", ".join("%s" for col in self.origin_visit_get_cols), updates=", ".join( "{0}=excluded.{0}".format(col) for col in self.origin_visit_get_cols ), ) cur.execute( query, ( origin_id, ov.visit, ov.date, ov.type, ov.status, ov.metadata, > ov.snapshot, ), ) E psycopg2.errors.NotNullViolation: null value in column "status" violates not-null constraint E DETAIL: Failing row contains (1, 1, 2013-05-07 04:20:39.369271+00, git, null, null, null). .tox/py3/lib/python3.7/site-packages/swh/storage/db.py:505: NotNullViolation