Changeset View
Changeset View
Standalone View
Standalone View
swh/provenance/algos/directory.py
| Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | def directory_flatten_range( | ||||
| start_id: Sha1Git, | start_id: Sha1Git, | ||||
| end_id: Sha1Git, | end_id: Sha1Git, | ||||
| minsize: int = 0, | minsize: int = 0, | ||||
| commit: bool = True, | commit: bool = True, | ||||
| ) -> None: | ) -> None: | ||||
| """Flatten the known directories from ``start_id`` to ``end_id``.""" | """Flatten the known directories from ``start_id`` to ``end_id``.""" | ||||
| current = start_id | current = start_id | ||||
| while current < end_id: | while current < end_id: | ||||
| dirs = provenance.storage.directory_iter_not_flattenned( | dirs = provenance.storage.directory_iter_not_flattened( | ||||
| limit=100, start_id=current | limit=100, start_id=current | ||||
| ) | ) | ||||
| if not dirs: | if not dirs: | ||||
| break | break | ||||
| directory_add( | directory_add( | ||||
| provenance, archive, [DirectoryEntry(id=d) for d in dirs], minsize, commit | provenance, archive, [DirectoryEntry(id=d) for d in dirs], minsize, commit | ||||
| ) | ) | ||||
| current = dirs[-1] | current = dirs[-1] | ||||
| @statsd.timed(metric=REVISION_DURATION_METRIC, tags={"method": "add"}) | @statsd.timed(metric=REVISION_DURATION_METRIC, tags={"method": "add"}) | ||||
| def directory_add( | def directory_add( | ||||
| provenance: ProvenanceInterface, | provenance: ProvenanceInterface, | ||||
| archive: ArchiveInterface, | archive: ArchiveInterface, | ||||
| directories: List[DirectoryEntry], | directories: List[DirectoryEntry], | ||||
| minsize: int = 0, | minsize: int = 0, | ||||
| commit: bool = True, | commit: bool = True, | ||||
| ) -> None: | ) -> None: | ||||
| for directory in directories: | for directory in directories: | ||||
| # Only flatten directories that are present in the provenance model, but not | # Only flatten directories that are present in the provenance model, but not | ||||
| # flattenned yet. | # flattened yet. | ||||
| flattenned = provenance.directory_already_flattenned(directory) | flattened = provenance.directory_already_flattened(directory) | ||||
| if flattenned is not None and not flattenned: | if flattened is not None and not flattened: | ||||
| directory_flatten( | directory_flatten( | ||||
| provenance, | provenance, | ||||
| archive, | archive, | ||||
| directory, | directory, | ||||
| minsize=minsize, | minsize=minsize, | ||||
| ) | ) | ||||
| if commit: | if commit: | ||||
| provenance.flush() | provenance.flush() | ||||
| Show All 14 Lines | while stack: | ||||
| current, prefix = stack.pop() | current, prefix = stack.pop() | ||||
| current.retrieve_children(archive, minsize=minsize) | current.retrieve_children(archive, minsize=minsize) | ||||
| for f_child in current.files: | for f_child in current.files: | ||||
| # Add content to the directory with the computed prefix. | # Add content to the directory with the computed prefix. | ||||
| provenance.content_add_to_directory(directory, f_child, prefix) | provenance.content_add_to_directory(directory, f_child, prefix) | ||||
| for d_child in current.dirs: | for d_child in current.dirs: | ||||
| # Recursively walk the child directory. | # Recursively walk the child directory. | ||||
| stack.append((d_child, os.path.join(prefix, d_child.name))) | stack.append((d_child, os.path.join(prefix, d_child.name))) | ||||
| provenance.directory_flag_as_flattenned(directory) | provenance.directory_flag_as_flattened(directory) | ||||