diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -40,6 +40,8 @@ class PackageLoader: # Origin visit type (str) set by the loader visit_type = "" + # Every packet of revisions, flush objects to storage + flush_nb_revisions = 10 def __init__(self, url): """Loader's constructor. This raises exception if the minimal required @@ -319,6 +321,7 @@ load_exceptions = [] + nb_revisions: int = 0 for version in self.get_versions(): # for each logger.debug("version: %s", version) tmp_revisions[version] = [] @@ -341,6 +344,13 @@ if revision_id is None: continue + nb_revisions += 1 + if (nb_revisions % self.flush_nb_revisions) == 0: + # first flush writing objects to storage + self.storage.flush() + # then clear remaining state from proxies (filter proxy) + self.storage.clear_buffers() + tmp_revisions[version].append((branch_name, revision_id)) if load_exceptions: