Changeset View
Changeset View
Standalone View
Standalone View
swh/graph/webgraph.py
Show All 36 Lines | class CompressionStep(Enum): | ||||
TRANSPOSE_OBL = 14 | TRANSPOSE_OBL = 14 | ||||
MAPS = 15 | MAPS = 15 | ||||
EXTRACT_PERSONS = 16 | EXTRACT_PERSONS = 16 | ||||
MPH_PERSONS = 17 | MPH_PERSONS = 17 | ||||
NODE_PROPERTIES = 18 | NODE_PROPERTIES = 18 | ||||
MPH_LABELS = 19 | MPH_LABELS = 19 | ||||
FCL_LABELS = 20 | FCL_LABELS = 20 | ||||
EDGE_LABELS = 21 | EDGE_LABELS = 21 | ||||
CLEAN_TMP = 22 | EDGE_LABELS_OBL = 22 | ||||
EDGE_LABELS_TRANSPOSE_OBL = 23 | |||||
CLEAN_TMP = 24 | |||||
def __str__(self): | def __str__(self): | ||||
return self.name | return self.name | ||||
# full compression pipeline | # full compression pipeline | ||||
COMP_SEQ = list(CompressionStep) | COMP_SEQ = list(CompressionStep) | ||||
▲ Show 20 Lines • Show All 172 Lines • ▼ Show 20 Lines | STEP_ARGV: Dict[CompressionStep, List[str]] = { | ||||
CompressionStep.EDGE_LABELS: [ | CompressionStep.EDGE_LABELS: [ | ||||
"{java}", | "{java}", | ||||
"org.softwareheritage.graph.compress.LabelMapBuilder", | "org.softwareheritage.graph.compress.LabelMapBuilder", | ||||
"--temp-dir", | "--temp-dir", | ||||
"{tmp_dir}", | "{tmp_dir}", | ||||
"{in_dir}", | "{in_dir}", | ||||
"{out_dir}/{graph_name}", | "{out_dir}/{graph_name}", | ||||
], | ], | ||||
CompressionStep.EDGE_LABELS_OBL: [ | |||||
"{java}", | |||||
"it.unimi.dsi.big.webgraph.labelling.BitStreamArcLabelledImmutableGraph", | |||||
"--list", | |||||
"{out_dir}/{graph_name}-labelled", | |||||
], | |||||
CompressionStep.EDGE_LABELS_TRANSPOSE_OBL: [ | |||||
"{java}", | |||||
"it.unimi.dsi.big.webgraph.labelling.BitStreamArcLabelledImmutableGraph", | |||||
"--list", | |||||
"{out_dir}/{graph_name}-transposed-labelled", | |||||
], | |||||
CompressionStep.CLEAN_TMP: [ | CompressionStep.CLEAN_TMP: [ | ||||
"rm", | "rm", | ||||
"-rf", | "-rf", | ||||
"{out_dir}/{graph_name}-base.graph", | "{out_dir}/{graph_name}-base.graph", | ||||
"{out_dir}/{graph_name}-base.offsets", | "{out_dir}/{graph_name}-base.offsets", | ||||
"{out_dir}/{graph_name}-base.properties", | "{out_dir}/{graph_name}-base.properties", | ||||
"{out_dir}/{graph_name}-bfs-simplified.graph", | "{out_dir}/{graph_name}-bfs-simplified.graph", | ||||
"{out_dir}/{graph_name}-bfs-simplified.offsets", | "{out_dir}/{graph_name}-bfs-simplified.offsets", | ||||
▲ Show 20 Lines • Show All 115 Lines • Show Last 20 Lines |