Changeset View
Changeset View
Standalone View
Standalone View
swh/graph/luigi.py
Show First 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | def run(self): | ||||
if self._export_meta().exists(): | if self._export_meta().exists(): | ||||
self._export_meta().remove() | self._export_meta().remove() | ||||
if self._compression_meta().exists(): | if self._compression_meta().exists(): | ||||
self._compression_meta().remove() | self._compression_meta().remove() | ||||
output_directory = self.local_graph_path | output_directory = self.local_graph_path | ||||
graph_name = "graph" | graph_name = "graph" | ||||
def progress_cb(percentage: int, step: webgraph.CompressionStep): | |||||
self.set_progress_percentage(percentage) | |||||
self.set_status_message(f"Running {step.name} (step #{step.value})") | |||||
start_date = datetime.datetime.now(tz=datetime.timezone.utc) | start_date = datetime.datetime.now(tz=datetime.timezone.utc) | ||||
webgraph.compress( | webgraph.compress( | ||||
graph_name, | graph_name, | ||||
self.local_export_path / "orc", | self.local_export_path / "orc", | ||||
output_directory, | output_directory, | ||||
steps, | steps, | ||||
conf, | conf, | ||||
) | ) | ||||
Show All 30 Lines |