diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -698,7 +698,7 @@ raise TypeError( 'snapshot_add expects one argument (or, as a legacy ' 'behavior, three arguments), not two') - if isinstance(snapshots, (int, bytes)): + if isinstance(snapshots, (int, str)): # Called by legacy code that uses the new api/client.py (origin, visit, snapshots) = \ (snapshots, origin, [visit]) @@ -815,7 +815,10 @@ branches. """ if isinstance(origin, int): - origin = self.origin_get({'id': origin})['url'] + origin = self.origin_get({'id': origin}) + if not origin: + return + origin = origin['url'] visit = self.origin_visit_get_latest( origin, allowed_statuses=allowed_statuses, require_snapshot=True) @@ -1274,6 +1277,12 @@ for visit in visits: if isinstance(visit['date'], str): visit['date'] = dateutil.parser.parse(visit['date']) + if isinstance(visit['origin'], str): + origin = \ + self.origin_get([{'url': visit['origin']}])[0] + if not origin: + raise ValueError('Unknown origin: %s' % visit['origin']) + visit['origin'] = origin['id'] if self.journal_writer: for visit in visits: @@ -1291,6 +1300,10 @@ while len(self._origin_visits[origin_id-1]) < visit_id: self._origin_visits[origin_id-1].append(None) + + visit = visit.copy() + visit['origin'] = origin_id + visit = self._origin_visits[origin_id-1][visit_id-1] = visit def origin_visit_get(self, origin, last_visit=None, limit=None): @@ -1337,7 +1350,10 @@ A visit. """ - origin = self.origin_get([{'url': origin}])[0]['id'] + origin = self.origin_get([{'url': origin}])[0] + if not origin: + return + origin = origin['id'] if origin <= len(self._origin_visits): visits = self._origin_visits[origin-1] return min( @@ -1356,7 +1372,10 @@ """ if isinstance(origin, str): - origin = self.origin_get({'url': origin})['id'] + origin = self.origin_get({'url': origin}) + if not origin: + return + origin = origin['id'] origin_visit = None if origin <= len(self._origin_visits) and \ visit <= len(self._origin_visits[origin-1]): @@ -1389,7 +1408,10 @@ snapshot (Optional[sha1_git]): identifier of the snapshot associated to the visit """ - origin = self.origin_get({'url': origin})['id'] + origin = self.origin_get({'url': origin}) + if not origin: + return + origin = origin['id'] visits = self._origin_visits[origin-1] if allowed_statuses is not None: visits = [visit for visit in visits diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -923,7 +923,7 @@ raise TypeError( 'snapshot_add expects one argument (or, as a legacy ' 'behavior, three arguments), not two') - if isinstance(snapshots, int): + if isinstance(snapshots, (int, str)): # Called by legacy code that uses the new api/client.py (origin_id, visit_id, snapshots) = \ (snapshots, origin, [visit]) @@ -1064,7 +1064,10 @@ branches. """ if isinstance(origin, int): - origin = self.origin_get({'id': origin}, db=db, cur=cur)['url'] + origin = self.origin_get({'id': origin}, db=db, cur=cur) + if not origin: + return + origin = origin['url'] origin_visit = self.origin_visit_get_latest( origin, allowed_statuses=allowed_statuses, require_snapshot=True, @@ -1284,6 +1287,9 @@ for visit in visits: if isinstance(visit['date'], str): visit['date'] = dateutil.parser.parse(visit['date']) + if isinstance(visit['origin'], str): + visit['origin'] = \ + self.origin_get({'url': visit['origin']})['id'] if self.journal_writer: for visit in visits: @@ -1306,7 +1312,7 @@ """Retrieve all the origin's visit's information. Args: - origin (int): The occurrence's origin (identifier). + origin (Union[int,str]): The occurrence's origin (identifier/URL). last_visit: Starting point from which listing the next visits Default to None limit (int): Number of results to return from the last visit. @@ -1340,7 +1346,10 @@ A visit. """ - origin = self.origin_get([{'url': origin}], db=db, cur=cur)[0]['id'] + origin = self.origin_get([{'url': origin}], db=db, cur=cur)[0] + if not origin: + return + origin = origin['id'] line = db.origin_visit_find_by_date(origin, visit_date, cur=cur) if line: return dict(zip(db.origin_visit_get_cols, line)) @@ -1358,7 +1367,10 @@ """ if isinstance(origin, str): - origin = self.origin_get({'url': origin}, db=db, cur=cur)['id'] + origin = self.origin_get({'url': origin}, db=db, cur=cur) + if not origin: + return + origin = origin['id'] ori_visit = db.origin_visit_get(origin, visit, cur) if not ori_visit: return None @@ -1393,7 +1405,10 @@ snapshot (Optional[sha1_git]): identifier of the snapshot associated to the visit """ - origin = self.origin_get({'url': origin}, db=db, cur=cur)['id'] + origin = self.origin_get({'url': origin}, db=db, cur=cur) + if not origin: + return + origin = origin['id'] origin_visit = db.origin_visit_get_latest( origin, allowed_statuses=allowed_statuses, @@ -1630,8 +1645,11 @@ of the added fetch_history entry """ if isinstance(origin_id, str): - origin_id = \ - self.origin_get([{'url': origin_id}], db=db, cur=cur)[0]['id'] + origin = \ + self.origin_get([{'url': origin_id}], db=db, cur=cur) + if not origin: + return + origin_id = origin[0]['id'] fetch_history = { 'origin': origin_id, 'date': datetime.datetime.now(tz=datetime.timezone.utc), diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -1599,7 +1599,14 @@ [('origin', expected_origin), ('origin_visit', data)]) - def test_origin_visit_add_default_type(self): + def test_origin_visit_get__unknown_origin(self): + self.assertEqual([], list(self.storage.origin_visit_get('foo'))) + self.assertEqual([], list(self.storage.origin_visit_get(10))) + + @given(strategies.booleans()) + def test_origin_visit_add_default_type(self, use_url): + self.reset_storage() + # given self.assertIsNone(self.storage.origin_get([self.origin2])[0]) @@ -1608,17 +1615,18 @@ # when origin_visit1 = self.storage.origin_visit_add( - origin_id, + self.origin2['url'] if use_url else origin_id, date=self.date_visit2) origin_visit2 = self.storage.origin_visit_add( - origin_id, + self.origin2['url'] if use_url else origin_id, date='2018-01-01 23:00:00+00') # then self.assertEqual(origin_visit1['origin'], origin_id) self.assertIsNotNone(origin_visit1['visit']) - actual_origin_visits = list(self.storage.origin_visit_get(origin_id)) + actual_origin_visits = list(self.storage.origin_visit_get( + self.origin2['url'] if use_url else origin_id)) self.assertEqual(actual_origin_visits, [ { 'origin': origin_id, @@ -1664,21 +1672,24 @@ ('origin_visit', data1), ('origin_visit', data2)]) - def test_origin_visit_update(self): + @given(strategies.booleans()) + def test_origin_visit_update(self, use_url): + self.reset_storage() + # given origin_id = self.storage.origin_add_one(self.origin) origin_id2 = self.storage.origin_add_one(self.origin2) origin_visit1 = self.storage.origin_visit_add( - origin_id, + self.origin['url'] if use_url else origin_id, date=self.date_visit2) origin_visit2 = self.storage.origin_visit_add( - origin_id, + self.origin['url'] if use_url else origin_id, date=self.date_visit3) origin_visit3 = self.storage.origin_visit_add( - origin_id2, + self.origin2['url'] if use_url else origin_id2, date=self.date_visit3) # when @@ -1687,13 +1698,16 @@ 'directories': 22, } self.storage.origin_visit_update( - origin_id, origin_visit1['visit'], status='full', + self.origin['url'] if use_url else origin_id, + origin_visit1['visit'], status='full', metadata=visit1_metadata) - self.storage.origin_visit_update(origin_id2, origin_visit3['visit'], - status='partial') + self.storage.origin_visit_update( + self.origin2['url'] if use_url else origin_id2, + origin_visit3['visit'], status='partial') # then - actual_origin_visits = list(self.storage.origin_visit_get(origin_id)) + actual_origin_visits = list(self.storage.origin_visit_get( + self.origin['url'] if use_url else origin_id)) self.assertEqual(actual_origin_visits, [{ 'origin': origin_visit2['origin'], 'date': self.date_visit2, @@ -1713,7 +1727,8 @@ }]) actual_origin_visits_bis = list(self.storage.origin_visit_get( - origin_id, limit=1)) + self.origin['url'] if use_url else origin_id, + limit=1)) self.assertEqual(actual_origin_visits_bis, [{ 'origin': origin_visit2['origin'], @@ -1726,7 +1741,8 @@ }]) actual_origin_visits_ter = list(self.storage.origin_visit_get( - origin_id, last_visit=origin_visit1['visit'])) + self.origin['url'] if use_url else origin_id, + last_visit=origin_visit1['visit'])) self.assertEqual(actual_origin_visits_ter, [{ 'origin': origin_visit2['origin'], @@ -1738,7 +1754,8 @@ 'snapshot': None, }]) - actual_origin_visits2 = list(self.storage.origin_visit_get(origin_id2)) + actual_origin_visits2 = list(self.storage.origin_visit_get( + self.origin2['url'] if use_url else origin_id2)) self.assertEqual(actual_origin_visits2, [{ 'origin': origin_visit3['origin'], @@ -1832,42 +1849,57 @@ self.origin['url'], self.date_visit2) self.assertEqual(visit['visit'], origin_visit3['visit']) - def test_origin_visit_update_missing_snapshot(self): + def test_origin_visit_find_by_date__unknown_origin(self): + self.storage.origin_visit_find_by_date('foo', self.date_visit2) + + @given(strategies.booleans()) + def test_origin_visit_update_missing_snapshot(self, use_url): + self.reset_storage() + # given origin_id = self.storage.origin_add_one(self.origin) origin_visit = self.storage.origin_visit_add( - origin_id, + self.origin['url'] if use_url else origin_id, date=self.date_visit1) # when self.storage.origin_visit_update( - origin_id, origin_visit['visit'], + self.origin['url'] if use_url else origin_id, + origin_visit['visit'], snapshot=self.snapshot['id']) # then actual_origin_visit = self.storage.origin_visit_get_by( - origin_visit['origin'], origin_visit['visit']) + self.origin['url'] if use_url else origin_id, + origin_visit['visit']) self.assertEqual(actual_origin_visit['snapshot'], self.snapshot['id']) # when self.storage.snapshot_add([self.snapshot]) self.assertEqual(actual_origin_visit['snapshot'], self.snapshot['id']) - def test_origin_visit_get_by(self): + @given(strategies.booleans()) + def test_origin_visit_get_by(self, use_url): + self.reset_storage() + origin_id = self.storage.origin_add_one(self.origin) origin_id2 = self.storage.origin_add_one(self.origin2) origin_visit1 = self.storage.origin_visit_add( - origin_id, + self.origin['url'] if use_url else origin_id, date=self.date_visit2) self.storage.snapshot_add(origin_id, origin_visit1['visit'], self.snapshot) # Add some other {origin, visit} entries - self.storage.origin_visit_add(origin_id, date=self.date_visit3) - self.storage.origin_visit_add(origin_id2, date=self.date_visit3) + self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + date=self.date_visit3) + self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id2, + date=self.date_visit3) # when visit1_metadata = { @@ -1876,7 +1908,8 @@ } self.storage.origin_visit_update( - origin_id, origin_visit1['visit'], status='full', + self.origin['url'] if use_url else origin_id, + origin_visit1['visit'], status='full', metadata=visit1_metadata) expected_origin_visit = origin_visit1.copy() @@ -1892,12 +1925,20 @@ # when actual_origin_visit1 = self.storage.origin_visit_get_by( - origin_visit1['origin'], origin_visit1['visit']) + self.origin['url'] if use_url else origin_id, + origin_visit1['visit']) # then self.assertEqual(actual_origin_visit1, expected_origin_visit) - def test_origin_visit_upsert_new(self): + def test_origin_visit_get_by__unknown_origin(self): + self.assertIsNone(self.storage.origin_visit_get_by(2, 10)) + self.assertIsNone(self.storage.origin_visit_get_by('foo', 10)) + + @given(strategies.booleans()) + def test_origin_visit_upsert_new(self, use_url): + self.reset_storage() + # given self.assertIsNone(self.storage.origin_get([self.origin2])[0]) @@ -1907,7 +1948,7 @@ # when self.storage.origin_visit_upsert([ { - 'origin': origin_id, + 'origin': self.origin2['url'] if use_url else origin_id, 'date': self.date_visit2, 'visit': 123, 'type': self.origin2['type'], @@ -1916,7 +1957,7 @@ 'snapshot': None, }, { - 'origin': origin_id, + 'origin': self.origin2['url'] if use_url else origin_id, 'date': '2018-01-01 23:00:00+00', 'visit': 1234, 'type': self.origin2['type'], @@ -1927,7 +1968,8 @@ ]) # then - actual_origin_visits = list(self.storage.origin_visit_get(origin_id)) + actual_origin_visits = list(self.storage.origin_visit_get( + self.origin2['url'] if use_url else origin_id)) self.assertEqual(actual_origin_visits, [ { 'origin': origin_id, @@ -1973,7 +2015,10 @@ ('origin_visit', data1), ('origin_visit', data2)]) - def test_origin_visit_upsert_existing(self): + @given(strategies.booleans()) + def test_origin_visit_upsert_existing(self, use_url): + self.reset_storage() + # given self.assertIsNone(self.storage.origin_get([self.origin2])[0]) @@ -1982,10 +2027,10 @@ # when origin_visit1 = self.storage.origin_visit_add( - origin_id, + self.origin2['url'] if use_url else origin_id, date=self.date_visit2) self.storage.origin_visit_upsert([{ - 'origin': origin_id, + 'origin': self.origin2['url'] if use_url else origin_id, 'date': self.date_visit2, 'visit': origin_visit1['visit'], 'type': self.origin2['type'], @@ -1998,7 +2043,8 @@ self.assertEqual(origin_visit1['origin'], origin_id) self.assertIsNotNone(origin_visit1['visit']) - actual_origin_visits = list(self.storage.origin_visit_get(origin_id)) + actual_origin_visits = list(self.storage.origin_visit_get( + self.origin2['url'] if use_url else origin_id)) self.assertEqual(actual_origin_visits, [{ 'origin': origin_id, @@ -2035,10 +2081,13 @@ ('origin_visit', data2)]) def test_origin_visit_get_by_no_result(self): - # No result actual_origin_visit = self.storage.origin_visit_get_by( 10, 999) + self.assertIsNone(actual_origin_visit) + self.storage.origin_add([self.origin]) + actual_origin_visit = self.storage.origin_visit_get_by( + self.origin['url'], 999) self.assertIsNone(actual_origin_visit) def test_person_get(self): @@ -2079,19 +2128,25 @@ expected_persons.reverse() self.assertEqual(list(actual_persons), expected_persons) - def test_origin_visit_get_latest(self): + @given(strategies.booleans()) + def test_origin_visit_get_latest(self, use_url): + self.reset_storage() + origin_id = self.storage.origin_add_one(self.origin) origin_url = self.origin['url'] - origin_visit1 = self.storage.origin_visit_add(origin_id, - self.date_visit1) + origin_visit1 = self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + self.date_visit1) visit1_id = origin_visit1['visit'] - origin_visit2 = self.storage.origin_visit_add(origin_id, - self.date_visit2) + origin_visit2 = self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + self.date_visit2) visit2_id = origin_visit2['visit'] # Add a visit with the same date as the previous one - origin_visit3 = self.storage.origin_visit_add(origin_id, - self.date_visit2) + origin_visit3 = self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + self.date_visit2) visit3_id = origin_visit3['visit'] origin_visit1 = self.storage.origin_visit_get_by(origin_url, visit1_id) @@ -2110,7 +2165,8 @@ # visit1 and require_snapshot=False still returns visit2 self.storage.snapshot_add([self.complete_snapshot]) self.storage.origin_visit_update( - origin_id, visit1_id, snapshot=self.complete_snapshot['id']) + self.origin['url'] if use_url else origin_id, + visit1_id, snapshot=self.complete_snapshot['id']) self.assertEqual( {**origin_visit1, 'snapshot': self.complete_snapshot['id']}, self.storage.origin_visit_get_latest( @@ -2129,7 +2185,9 @@ ) # Mark the first visit as completed and check status filter again - self.storage.origin_visit_update(origin_id, visit1_id, status='full') + self.storage.origin_visit_update( + self.origin['url'] if use_url else origin_id, + visit1_id, status='full') self.assertEqual( { **origin_visit1, @@ -2146,7 +2204,8 @@ # Add snapshot to visit2 and check that the new snapshot is returned self.storage.snapshot_add([self.empty_snapshot]) self.storage.origin_visit_update( - origin_id, visit2_id, snapshot=self.empty_snapshot['id']) + self.origin['url'] if use_url else origin_id, + visit2_id, snapshot=self.empty_snapshot['id']) self.assertEqual( {**origin_visit2, 'snapshot': self.empty_snapshot['id']}, self.storage.origin_visit_get_latest( @@ -2170,7 +2229,8 @@ # Add snapshot to visit3 (same date as visit2) self.storage.snapshot_add([self.complete_snapshot]) self.storage.origin_visit_update( - origin_id, visit3_id, snapshot=self.complete_snapshot['id']) + self.origin['url'] if use_url else origin_id, + visit3_id, snapshot=self.complete_snapshot['id']) self.assertEqual( { **origin_visit1, @@ -2825,113 +2885,154 @@ self.storage.snapshot_get_latest( self.origin['url'] if use_url else origin_id)) - def test_snapshot_get_latest__missing_snapshot(self): + @given(strategies.booleans()) + def test_snapshot_get_latest__missing_snapshot(self, use_url): + self.reset_storage() + + # Origin does not exist + self.assertIsNone(self.storage.snapshot_get_latest( + self.origin['url'] if use_url else 999)) + origin_id = self.storage.origin_add_one(self.origin) - origin_visit1 = self.storage.origin_visit_add(origin_id, - self.date_visit1) + origin_visit1 = self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + self.date_visit1) visit1_id = origin_visit1['visit'] - origin_visit2 = self.storage.origin_visit_add(origin_id, - self.date_visit2) + origin_visit2 = self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + self.date_visit2) visit2_id = origin_visit2['visit'] # Two visits, both with no snapshot: latest snapshot is None - self.assertIsNone(self.storage.snapshot_get_latest(origin_id)) + self.assertIsNone(self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id)) # Add unknown snapshot to visit1, check that the inconsistency is # detected self.storage.origin_visit_update( - origin_id, visit1_id, snapshot=self.complete_snapshot['id']) + self.origin['url'] if use_url else origin_id, + visit1_id, snapshot=self.complete_snapshot['id']) with self.assertRaises(ValueError): - self.storage.snapshot_get_latest(origin_id) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id) # Status filter: both visits are status=ongoing, so no snapshot # returned self.assertIsNone( - self.storage.snapshot_get_latest(origin_id, - allowed_statuses=['full']) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id, + allowed_statuses=['full']) ) # Mark the first visit as completed and check status filter again - self.storage.origin_visit_update(origin_id, visit1_id, status='full') + self.storage.origin_visit_update( + self.origin['url'] if use_url else origin_id, + visit1_id, status='full') with self.assertRaises(ValueError): - self.storage.snapshot_get_latest(origin_id, - allowed_statuses=['full']), + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id, + allowed_statuses=['full']), # Actually add the snapshot and check status filter again self.storage.snapshot_add([self.complete_snapshot]) self.assertEqual( self.complete_snapshot, - self.storage.snapshot_get_latest(origin_id) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id) ) # Add unknown snapshot to visit2 and check that the inconsistency # is detected self.storage.origin_visit_update( - origin_id, visit2_id, snapshot=self.snapshot['id']) + self.origin['url'] if use_url else origin_id, + visit2_id, snapshot=self.snapshot['id']) with self.assertRaises(ValueError): - self.storage.snapshot_get_latest(origin_id) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id) # Actually add that snapshot and check that the new one is returned self.storage.snapshot_add([self.snapshot]) self.assertEqual( self.snapshot, - self.storage.snapshot_get_latest(origin_id) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id) ) - def test_snapshot_get_latest__legacy_add(self): + @given(strategies.booleans()) + def test_snapshot_get_latest__legacy_add(self, use_url): + self.reset_storage() + origin_id = self.storage.origin_add_one(self.origin) - origin_visit1 = self.storage.origin_visit_add(origin_id, - self.date_visit1) + origin_visit1 = self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + self.date_visit1) visit1_id = origin_visit1['visit'] - origin_visit2 = self.storage.origin_visit_add(origin_id, - self.date_visit2) + origin_visit2 = self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + self.date_visit2) visit2_id = origin_visit2['visit'] # Add a visit with the same date as the previous one - origin_visit3 = self.storage.origin_visit_add(origin_id, - self.date_visit2) + origin_visit3 = self.storage.origin_visit_add( + self.origin['url'] if use_url else origin_id, + self.date_visit2) visit3_id = origin_visit3['visit'] # Two visits, both with no snapshot: latest snapshot is None - self.assertIsNone(self.storage.snapshot_get_latest(origin_id)) + self.assertIsNone(self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id)) # Add snapshot to visit1, latest snapshot = visit 1 snapshot - self.storage.snapshot_add(origin_id, visit1_id, self.complete_snapshot) + self.storage.snapshot_add( + self.origin['url'] if use_url else origin_id, + visit1_id, self.complete_snapshot) self.assertEqual(self.complete_snapshot, - self.storage.snapshot_get_latest(origin_id)) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id)) # Status filter: both visits are status=ongoing, so no snapshot # returned self.assertIsNone( - self.storage.snapshot_get_latest(origin_id, - allowed_statuses=['full']) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id, + allowed_statuses=['full']) ) # Mark the first visit as completed and check status filter again - self.storage.origin_visit_update(origin_id, visit1_id, status='full') + self.storage.origin_visit_update( + self.origin['url'] if use_url else origin_id, + visit1_id, status='full') self.assertEqual( self.complete_snapshot, - self.storage.snapshot_get_latest(origin_id, - allowed_statuses=['full']), + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id, + allowed_statuses=['full']), ) # Add snapshot to visit2 and check that the new snapshot is returned - self.storage.snapshot_add(origin_id, visit2_id, self.empty_snapshot) + self.storage.snapshot_add( + self.origin['url'] if use_url else origin_id, + visit2_id, self.empty_snapshot) self.assertEqual(self.empty_snapshot, - self.storage.snapshot_get_latest(origin_id)) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id)) # Check that the status filter is still working self.assertEqual( self.complete_snapshot, - self.storage.snapshot_get_latest(origin_id, - allowed_statuses=['full']), + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id, + allowed_statuses=['full']), ) # Add snapshot to visit3 (same date as visit2) and check that # the new snapshot is returned - self.storage.snapshot_add(origin_id, visit3_id, self.complete_snapshot) + self.storage.snapshot_add( + self.origin['url'] if use_url else origin_id, + visit3_id, self.complete_snapshot) self.assertEqual(self.complete_snapshot, - self.storage.snapshot_get_latest(origin_id)) + self.storage.snapshot_get_latest( + self.origin['url'] if use_url else origin_id)) def test_stat_counters(self): expected_keys = ['content', 'directory', @@ -2960,12 +3061,11 @@ # Add other objects. Check their counter increased as well. - origin_id = self.storage.origin_add_one(self.origin2) + self.storage.origin_add_one(self.origin2) origin_visit1 = self.storage.origin_visit_add( - origin_id, - date=self.date_visit2) - self.storage.snapshot_add(origin_id, origin_visit1['visit'], - self.snapshot) + self.origin2['url'], date=self.date_visit2) + self.storage.snapshot_add( + self.origin2['url'], origin_visit1['visit'], self.snapshot) self.storage.directory_add([self.dir]) self.storage.revision_add([self.revision]) self.storage.release_add([self.release]) @@ -3391,7 +3491,8 @@ def test_origin_metadata_add(self): # given origin_id = self.storage.origin_add([self.origin])[0]['id'] - origin_metadata0 = list(self.storage.origin_metadata_get_by(origin_id)) + origin_metadata0 = list(self.storage.origin_metadata_get_by( + origin_id)) self.assertTrue(len(origin_metadata0) == 0) tools = self.storage.tool_add([self.metadata_tool]) @@ -3420,7 +3521,8 @@ provider['id'], tool['id'], self.origin_metadata2['metadata']) - actual_om = list(self.storage.origin_metadata_get_by(origin_id)) + actual_om = list(self.storage.origin_metadata_get_by( + origin_id)) # then self.assertCountEqual( [item['origin_id'] for item in actual_om], @@ -3459,9 +3561,10 @@ provider['id'], tool['id'], self.origin_metadata2['metadata']) - all_metadatas = list(self.storage.origin_metadata_get_by(origin_id)) + all_metadatas = list(self.storage.origin_metadata_get_by( + origin_id)) metadatas_for_origin2 = list(self.storage.origin_metadata_get_by( - origin_id2)) + origin_id2)) expected_results = [{ 'origin_id': origin_id, 'discovery_date': datetime.datetime( @@ -3554,22 +3657,21 @@ # when adding for the same origin 2 metadatas self.storage.origin_metadata_add( - origin_id, - self.origin_metadata['discovery_date'], - provider1['id'], - tool['id'], - self.origin_metadata['metadata']) + origin_id, + self.origin_metadata['discovery_date'], + provider1['id'], + tool['id'], + self.origin_metadata['metadata']) self.storage.origin_metadata_add( - origin_id2, - self.origin_metadata2['discovery_date'], - provider2['id'], - tool['id'], - self.origin_metadata2['metadata']) + origin_id2, + self.origin_metadata2['discovery_date'], + provider2['id'], + tool['id'], + self.origin_metadata2['metadata']) provider_type = 'registry' - m_by_provider = list(self.storage. - origin_metadata_get_by( - origin_id2, - provider_type)) + m_by_provider = list(self.storage.origin_metadata_get_by( + origin_id2, + provider_type)) for item in m_by_provider: if 'id' in item: del item['id']