Page MenuHomeSoftware Heritage
Differential D422

Load snapshots without depending on the structure of the previous snapshot
ClosedPublic
Actions

Authored by olasd on Sep 19 2018, 2:04 PM.

Details

Reviewers
ardumont
Commits
rDLDPYd27a7b2e4ab6: Load snapshots without depending on the structure of the previous snapshot
Summary

This makes sure we account for potential removals of versions from one visit to
the next.

Depends on D421

Test Plan

Updated the "revision unpublished" test case to match the new snapshot

Diff Detail

Repository
rDLDPY PyPI loader
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathSize
swh/
loader/
pypi/
54 lines
tests/
34 lines

Diff 1309

View Options

swh/loader/pypi/loader.py

Show First 20 Lines • Show All 100 Lines • ▼ Show 20 Lines def _known_artifacts(self, last_snapshot):
Args Args
snapshot (dict): Last snapshot for the visit snapshot (dict): Last snapshot for the visit
Returns: Returns:
list of (filename, sha256) tuples. list of (filename, sha256) tuples.
""" """
if not last_snapshot or 'branches' not in last_snapshot:
return {}
revs = [rev['target'] for rev in last_snapshot['branches'].values()] revs = [rev['target'] for rev in last_snapshot['branches'].values()]
known_revisions = self.storage.revision_get(revs) known_revisions = self.storage.revision_get(revs)
ret = [] ret = {}
for revision in known_revisions: for revision in known_revisions:
if 'original_artifact' in revision['metadata']: if 'original_artifact' in revision['metadata']:
artifact = revision['metadata']['original_artifact'] artifact = revision['metadata']['original_artifact']
ret.append((artifact['filename'], artifact['sha256'])) ret[artifact['filename'], artifact['sha256']] = revision['id']
return ret return ret
def _last_snapshot(self): def _last_snapshot(self):
"""Retrieve the last snapshot """Retrieve the last snapshot
""" """
return self.storage.snapshot_get_latest(self.origin_id) return self.storage.snapshot_get_latest(self.origin_id)
Show All 17 Linesclass PyPILoader(SWHLoader):
def _prepare_state(self): def _prepare_state(self):
"""Initialize internal state (snapshot, contents, directories, etc...) """Initialize internal state (snapshot, contents, directories, etc...)
This is called from `prepare` method. This is called from `prepare` method.
""" """
last_snapshot = self._last_snapshot() last_snapshot = self._last_snapshot()
if last_snapshot: self.known_artifacts = self._known_artifacts(last_snapshot)
self._snapshot = last_snapshot.copy()
known_artifacts = self._known_artifacts(self._snapshot)
else:
self._snapshot = {
'branches': {}
}
known_artifacts = []
# and the artifacts # and the artifacts
# that will be the source of data to retrieve # that will be the source of data to retrieve
self.release_artifacts = self.project.download_new_releases( self.new_artifacts = self.project.download_new_releases(
known_artifacts self.known_artifacts
) )
# temporary state # temporary state
self._contents = [] self._contents = []
self._directories = [] self._directories = []
self._revisions = [] self._revisions = []
def fetch_data(self): def fetch_data(self):
"""Called once per release artifact version (can be many for one """Called once per release artifact version (can be many for one
release). release).
This will for each call: This will for each call:
- retrieve a release artifact (associated to a release version) - retrieve a release artifact (associated to a release version)
- Uncompress it and compute the necessary information - Uncompress it and compute the necessary information
- Computes the swh objects - Computes the swh objects
Returns: Returns:
True as long as data to fetch exist True as long as data to fetch exist
""" """
data = None data = None
if self.done: if self.done:
return False return False
try: try:
data = next(self.release_artifacts) data = next(self.new_artifacts)
except StopIteration: except StopIteration:
self.done = True self.done = True
return False return False
project_info, author, release, artifact, dir_path = data project_info, author, release, artifact, dir_path = data
dir_path = dir_path.encode('utf-8') dir_path = dir_path.encode('utf-8')
directory = Directory.from_disk(path=dir_path, data=True) directory = Directory.from_disk(path=dir_path, data=True)
_objects = directory.collect() _objects = directory.collect()
Show All 24 Lines def fetch_data(self):
'directory': directory.hash, 'directory': directory.hash,
'parents': [], 'parents': [],
'type': 'tar', 'type': 'tar',
} }
_revision['id'] = identifier_to_bytes( _revision['id'] = identifier_to_bytes(
revision_identifier(_revision)) revision_identifier(_revision))
self._revisions.append(_revision) self._revisions.append(_revision)
branch_name = artifact['filename'].encode('utf-8') artifact_key = artifact['filename'], artifact['sha256']
self._snapshot['branches'][branch_name] = { self.known_artifacts[artifact_key] = _revision['id']
'target': _revision['id'],
'target_type': 'revision',
}
return not self.done return not self.done
def target_from_artifact(self, filename, sha256):
target = self.known_artifacts.get((filename, sha256))
if target:
return {
'target': target,
'target_type': 'revision',
}
return None
def generate_and_load_snapshot(self): def generate_and_load_snapshot(self):
self._snapshot['id'] = identifier_to_bytes( branches = {}
snapshot_identifier(self._snapshot)) for release, artifacts in self.project.all_release_artifacts().items():
self.maybe_load_snapshot(self._snapshot) for filename, sha256 in artifacts:
branch_name = filename
target = self.target_from_artifact(filename, sha256)
branches[branch_name.encode('utf-8')] = target
snapshot = {
'branches': branches,
}
snapshot['id'] = identifier_to_bytes(
snapshot_identifier(snapshot))
self.maybe_load_snapshot(snapshot)
def store_data(self): def store_data(self):
"""(override) This sends collected objects to storage. """(override) This sends collected objects to storage.
""" """
self.maybe_load_contents(self._contents) self.maybe_load_contents(self._contents)
self.maybe_load_directories(self._directories) self.maybe_load_directories(self._directories)
self.maybe_load_revisions(self._revisions) self.maybe_load_revisions(self._revisions)
Show All 20 Lines
View Options

swh/loader/pypi/tests/test_loader.py

Show First 20 Lines • Show All 211 Lines • ▼ Show 20 Lines def _last_snapshot(self):
b'\x9a\x07M!', b'\x9a\x07M!',
'target_type': 'revision' 'target_type': 'revision'
}, },
}, },
} }
def _known_artifacts(self, last_snapshot): def _known_artifacts(self, last_snapshot):
"""List corresponding seen release artifacts""" """List corresponding seen release artifacts"""
yield from [ return {
( (
'0805nexter-1.1.0.zip', '0805nexter-1.1.0.zip',
'52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035' # noqa '52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035' # noqa
), ): b'L\x99\x89\x1f\x93\xb8\x14P8Ww#Z7\xb5\xe9f\xdd\x15q',
( (
'0805nexter-1.2.0.zip', '0805nexter-1.2.0.zip',
'49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709' # noqa '49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709' # noqa
) ): b'\xe4E\xdaM\xa2+1\xbf\xeb\xb6\xff\xc48=\xbf\x83\x9a\x07M!',
] }
class LoaderNoNewChangesSinceLastVisitITest(BaseLoaderITest): class LoaderNoNewChangesSinceLastVisitITest(BaseLoaderITest):
"""This scenario makes use of the incremental nature of the loader. """This scenario makes use of the incremental nature of the loader.
If nothing changes in between visits, the snapshot for the visit If nothing changes in between visits, the snapshot for the visit
must stay the same as the first visit. must stay the same as the first visit.
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines def _last_snapshot(self):
b'0805nexter-1.3.0.zip': { b'0805nexter-1.3.0.zip': {
'target': b'\xfbF\xe4\x96\x05\xb0\xbb\xe6\x9f\x8cS\xd3\x15\xe8\x93p\xe7\xc6\xcb]', # noqa 'target': b'\xfbF\xe4\x96\x05\xb0\xbb\xe6\x9f\x8cS\xd3\x15\xe8\x93p\xe7\xc6\xcb]', # noqa
'target_type': 'revision' 'target_type': 'revision'
} }
} }
} }
def _known_artifacts(self, last_snapshot): def _known_artifacts(self, last_snapshot):
"""List corresponding seen release artifacts""" """Map previously seen release artifacts to their revision"""
return [ return {
( (
'0805nexter-1.1.0.zip', '0805nexter-1.1.0.zip',
'52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035' # noqa '52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035' # noqa
), ): b'L\x99\x89\x1f\x93\xb8\x14P8Ww#Z7\xb5\xe9f\xdd\x15q',
( (
'0805nexter-1.2.0.zip', '0805nexter-1.2.0.zip',
'49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709' # noqa '49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709' # noqa
), ): b'\xe4E\xdaM\xa2+1\xbf\xeb\xb6\xff\xc48=\xbf\x83\x9a\x07M!',
( (
'0805nexter-1.3.0.zip', '0805nexter-1.3.0.zip',
'7097c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1' # noqa '7097c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1' # noqa
) ): b'\xfbF\xe4\x96\x05\xb0\xbb\xe6\x9f\x8cS\xd3\x15\xe8\x93p\xe7\xc6\xcb]', # noqa
] }
class LoaderChangesOldReleaseArtifactRemovedSinceLastVisit(BaseLoaderITest): class LoaderChangesOldReleaseArtifactRemovedSinceLastVisit(BaseLoaderITest):
"""In this scenario, a visit has already taken place. An existing """In this scenario, a visit has already taken place. An existing
snapshot exists. snapshot exists.
The PyPI project has changed: The PyPI project has changed:
- a new release has been uploaded - a new release has been uploaded
- an older one has been removed - an older one has been removed
# What wrongly happens now
The visit results in a new snapshot, such snapshot shares the same
branches as last visit's snapshot (including the unpublished
release artifacts).
# What should happen:
The visit should result in a new snapshot. Such snapshot shares some of The visit should result in a new snapshot. Such snapshot shares some of
the same branches as prior visit (but not all): the same branches as prior visit (but not all):
- new release artifact branch exists - new release artifact branch exists
- old release artifact branch has been removed - old release artifact branch has been removed
- the other unchanged release artifact branches are left unchanged - the other unchanged release artifact branches are left unchanged
""" """
Show All 35 Lines def load(self):
expected_directories = [ expected_directories = [
'a2b7621f3e52eb3632657f6e3436bd08202db56f', # new one 'a2b7621f3e52eb3632657f6e3436bd08202db56f', # new one
'770e21215ecac53cea331d8ea4dc0ffc9d979367', '770e21215ecac53cea331d8ea4dc0ffc9d979367',
] ]
self.assertDirectoriesOk(expected_directories) self.assertDirectoriesOk(expected_directories)
expected_revisions = { expected_revisions = {
# 1.4.0
'5e91875f096ac48c98d74acf307439a3490f2827': '770e21215ecac53cea331d8ea4dc0ffc9d979367', # noqa '5e91875f096ac48c98d74acf307439a3490f2827': '770e21215ecac53cea331d8ea4dc0ffc9d979367', # noqa
} }
self.assertRevisionsOk(expected_revisions) self.assertRevisionsOk(expected_revisions)
old_revisions = { old_revisions = {
'4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa # 1.2.0
'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa 'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa
# 1.3.0
'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d': 'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a', # noqa 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d': 'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a', # noqa
} }
for rev, dir_id in old_revisions.items(): for rev, dir_id in old_revisions.items():
expected_revisions[rev] = dir_id expected_revisions[rev] = dir_id
expected_snapshot_id = '0f36392a2c825bafa766b371f2485f9549eec41c' expected_snapshot_id = 'fb192f35397812776377fa758e0ba4cf20a4cf5d'
self.assertSnapshotOk(expected_snapshot_id, expected_revisions) self.assertSnapshotOk(expected_snapshot_id, expected_revisions)
_id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id'])
self.assertNotEqual(expected_snapshot_id, _id) self.assertNotEqual(expected_snapshot_id, _id)
About · Developer information · Legal