diff --git a/bin/load-history-from-snapshot b/bin/load-history-from-snapshot index 1fddcc6..2dc6069 100755 --- a/bin/load-history-from-snapshot +++ b/bin/load-history-from-snapshot @@ -1,56 +1,57 @@ #!/usr/bin/python3 import glob import logging import os import sys from swh.loader.debian.listers.snapshot import SnapshotDebianOrg from swh.loader.debian.loader import ( process_source_packages, try_flush_partial, flush_release, flush_revision) from swh.storage.storage import Storage logging.basicConfig() log = logging.getLogger('swh.loader.debian.load_history_from_snapshot') keyrings = glob.glob('/usr/share/keyrings/*') s = SnapshotDebianOrg(connstr='service=snapshot', basedir=os.path.expanduser('~/tmp/snapshot.d.o')) source_package_dir = sys.argv[1] package_names = sys.argv[2:] pkgs = s.copy_package_files(package_names, source_package_dir, log=log) sorted_pkgs = sorted((p for p in pkgs.values() if os.path.exists(p['dsc'])), key=lambda p: (p['name'], p['version'])) storage = Storage('dbname=softwareheritage-dev', '/tmp/swh-loader-debian/objects') partial = {} for partial in process_source_packages(sorted_pkgs, keyrings, log=log): print( + partial['packages'][-1]['name'], partial['packages'][-1]['version'], len(partial['objects']['directory']), '(%s)' % len(partial['objects']['directory_seen']), len(partial['objects']['content']), '(%s)' % len(partial['objects']['content_seen']), ) try_flush_partial(storage, partial, content_packet_size=10000, content_packet_length=1024 * 1024 * 1024, content_max_length_one=100 * 1024 * 1024, directory_packet_size=25000, log=log) if partial: try_flush_partial(storage, partial, content_packet_size=10000, content_packet_length=1024 * 1024 * 1024, content_max_length_one=100 * 1024 * 1024, directory_packet_size=25000, force=True, log=log) packages = flush_revision(storage, partial) revisions = flush_release(storage, packages)