Page MenuHomeSoftware Heritage
Differential D427

Implement load_status
ClosedPublic
Actions

Authored by olasd on Sep 19 2018, 3:47 PM.

Details

Reviewers
ardumont
Commits
rDLDPY2f0b8cfcbfda: Implement load_status
Summary

The load is eventful if we have loaded new artifacts

Depends on D426

Test Plan

Integration tests have been updated to check for the new status

Diff Detail

Repository
rDLDPY PyPI loader
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathSize
swh/
loader/
pypi/
7 lines
tests/
8 lines

Diff 1320

View Options

swh/loader/pypi/loader.py

Show First 20 Lines • Show All 153 Lines • ▼ Show 20 Lines def _prepare_state(self):
# that will be the source of data to retrieve # that will be the source of data to retrieve
self.new_artifacts = self.project.download_new_releases( self.new_artifacts = self.project.download_new_releases(
self.known_artifacts self.known_artifacts
) )
# temporary state # temporary state
self._contents = [] self._contents = []
self._directories = [] self._directories = []
self._revisions = [] self._revisions = []
self._load_status = 'uneventful'
def fetch_data(self): def fetch_data(self):
"""Called once per release artifact version (can be many for one """Called once per release artifact version (can be many for one
release). release).
This will for each call: This will for each call:
- retrieve a release artifact (associated to a release version) - retrieve a release artifact (associated to a release version)
- Uncompress it and compute the necessary information - Uncompress it and compute the necessary information
- Computes the swh objects - Computes the swh objects
Returns: Returns:
True as long as data to fetch exist True as long as data to fetch exist
""" """
data = None data = None
if self.done: if self.done:
return False return False
try: try:
data = next(self.new_artifacts) data = next(self.new_artifacts)
self._load_status = 'eventful'
except StopIteration: except StopIteration:
self.done = True self.done = True
return False return False
project_info, author, release, artifact, dir_path = data project_info, author, release, artifact, dir_path = data
dir_path = dir_path.encode('utf-8') dir_path = dir_path.encode('utf-8')
directory = Directory.from_disk(path=dir_path, data=True) directory = Directory.from_disk(path=dir_path, data=True)
_objects = directory.collect() _objects = directory.collect()
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines def store_data(self):
self.maybe_load_contents(self._contents) self.maybe_load_contents(self._contents)
self.maybe_load_directories(self._directories) self.maybe_load_directories(self._directories)
self.maybe_load_revisions(self._revisions) self.maybe_load_revisions(self._revisions)
if self.done: if self.done:
self.generate_and_load_snapshot() self.generate_and_load_snapshot()
self.flush() self.flush()
def load_status(self):
return {
'status': self._load_status,
}
if __name__ == '__main__': if __name__ == '__main__':
import logging import logging
import sys import sys
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
if len(sys.argv) != 2: if len(sys.argv) != 2:
logging.error('Usage: %s <module-name>' % sys.argv[0]) logging.error('Usage: %s <module-name>' % sys.argv[0])
sys.exit(1) sys.exit(1)
module_name = sys.argv[1] module_name = sys.argv[1]
loader = PyPILoader() loader = PyPILoader()
loader.load( loader.load(
module_name, module_name,
'https://pypi.org/projects/%s/' % module_name, 'https://pypi.org/projects/%s/' % module_name,
'https://pypi.org/pypi/%s/json' % module_name, 'https://pypi.org/pypi/%s/json' % module_name,
) )
View Options

swh/loader/pypi/tests/test_loader.py

Show First 20 Lines • Show All 200 Lines • ▼ Show 20 Lines def load(self):
'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21', 'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
'target_type': 'revision', 'target_type': 'revision',
}, },
} }
self.assertSnapshotOk('5c99339508de5eea67eecd3e0ba1efa8e804b96e', self.assertSnapshotOk('5c99339508de5eea67eecd3e0ba1efa8e804b96e',
expected_branches) expected_branches)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
class PyPILoaderWithSnapshot(TestPyPILoader): class PyPILoaderWithSnapshot(TestPyPILoader):
"""This loader provides a snapshot and lists corresponding seen """This loader provides a snapshot and lists corresponding seen
release artifacts. release artifacts.
""" """
def _last_snapshot(self): def _last_snapshot(self):
"""Return last visited snapshot""" """Return last visited snapshot"""
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines def load(self):
'target_type': 'revision', 'target_type': 'revision',
}, },
} }
self.assertSnapshotOk(expected_snapshot_id, expected_branches) self.assertSnapshotOk(expected_snapshot_id, expected_branches)
_id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id'])
self.assertEquals(expected_snapshot_id, _id) self.assertEquals(expected_snapshot_id, _id)
self.assertEqual(self.loader.load_status(), {'status': 'uneventful'})
class LoaderNewChangesSinceLastVisitITest(BaseLoaderITest): class LoaderNewChangesSinceLastVisitITest(BaseLoaderITest):
"""In this scenario, a visit has already taken place. """In this scenario, a visit has already taken place.
An existing snapshot exists. An existing snapshot exists.
This time, the PyPI project has changed, a new release (with 1 new This time, the PyPI project has changed, a new release (with 1 new
release artifact) has been uploaded. The old releases did not release artifact) has been uploaded. The old releases did not
change. change.
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines def load(self):
}, },
} }
self.assertSnapshotOk(expected_snapshot_id, expected_branches) self.assertSnapshotOk(expected_snapshot_id, expected_branches)
_id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id'])
self.assertNotEqual(expected_snapshot_id, _id) self.assertNotEqual(expected_snapshot_id, _id)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
class PyPILoaderWithSnapshot2(TestPyPILoader): class PyPILoaderWithSnapshot2(TestPyPILoader):
"""This loader provides a snapshot and lists corresponding seen """This loader provides a snapshot and lists corresponding seen
release artifacts. release artifacts.
""" """
def _last_snapshot(self): def _last_snapshot(self):
"""Return last visited snapshot""" """Return last visited snapshot"""
▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines def load(self):
'target': '5e91875f096ac48c98d74acf307439a3490f2827', 'target': '5e91875f096ac48c98d74acf307439a3490f2827',
'target_type': 'revision', 'target_type': 'revision',
}, },
} }
self.assertSnapshotOk(expected_snapshot_id, expected_branches) self.assertSnapshotOk(expected_snapshot_id, expected_branches)
_id = hashutil.hash_to_hex(self.loader._last_snapshot()['id']) _id = hashutil.hash_to_hex(self.loader._last_snapshot()['id'])
self.assertNotEqual(expected_snapshot_id, _id) self.assertNotEqual(expected_snapshot_id, _id)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
About · Developer information · Legal