Changeset View
Changeset View
Standalone View
Standalone View
swh/graph/luigi/origin_contributors.py
| Show All 14 Lines | |||||
| # control | # control | ||||
| from pathlib import Path | from pathlib import Path | ||||
| from typing import Dict, List, Tuple | from typing import Dict, List, Tuple | ||||
| import luigi | import luigi | ||||
| from .compressed_graph import LocalGraph | from .compressed_graph import LocalGraph | ||||
| from .misc_datasets import TopoSort | from .misc_datasets import TopoSort | ||||
| from .utils import run_script | from .utils import run_script, silence_webgraph_cache_warning | ||||
| class ListOriginContributors(luigi.Task): | class ListOriginContributors(luigi.Task): | ||||
| """Creates a file that contains all SWHIDs in topological order from a compressed | """Creates a file that contains all SWHIDs in topological order from a compressed | ||||
| graph.""" | graph.""" | ||||
| local_graph_path = luigi.PathParameter() | local_graph_path = luigi.PathParameter() | ||||
| topological_order_path = luigi.PathParameter() | topological_order_path = luigi.PathParameter() | ||||
| Show All 14 Lines | class ListOriginContributors(luigi.Task): | ||||
| def output(self) -> luigi.Target: | def output(self) -> luigi.Target: | ||||
| """.csv.zst file that contains the topological order.""" | """.csv.zst file that contains the topological order.""" | ||||
| return luigi.LocalTarget(self.origin_contributors_path) | return luigi.LocalTarget(self.origin_contributors_path) | ||||
| def run(self) -> None: | def run(self) -> None: | ||||
| """Runs org.softwareheritage.graph.utils.TopoSort and compresses""" | """Runs org.softwareheritage.graph.utils.TopoSort and compresses""" | ||||
| class_name = "org.softwareheritage.graph.utils.ListOriginContributors" | class_name = "org.softwareheritage.graph.utils.ListOriginContributors" | ||||
| silence_webgraph_cache_warning(self.local_graph_path) | |||||
| script = f""" | script = f""" | ||||
| zstdcat {self.topological_order_path} \ | zstdcat {self.topological_order_path} \ | ||||
| | java {class_name} '{self.local_graph_path}/{self.graph_name}' \ | | java {class_name} '{self.local_graph_path}/{self.graph_name}' \ | ||||
| | pv --line-mode --wait \ | | pv --line-mode --wait \ | ||||
| | zstdmt -19 | | zstdmt -19 | ||||
| """ | """ | ||||
| run_script(script, self.origin_contributors_path) | run_script(script, self.origin_contributors_path) | ||||
| ▲ Show 20 Lines • Show All 136 Lines • Show Last 20 Lines | |||||