Changeset View
Changeset View
Standalone View
Standalone View
swh/graph/webgraph.py
Show All 9 Lines | |||||
import logging | import logging | ||||
import os | import os | ||||
import subprocess | import subprocess | ||||
from enum import Enum | from enum import Enum | ||||
from datetime import datetime | from datetime import datetime | ||||
from pathlib import Path | from pathlib import Path | ||||
from typing import Dict, List, Set | from typing import Dict, List, Set | ||||
from click import ParamType | |||||
from swh.graph.config import check_config_compress | from swh.graph.config import check_config_compress | ||||
class CompressionStep(Enum): | class CompressionStep(Enum): | ||||
MPH = 1 | MPH = 1 | ||||
BV = 2 | BV = 2 | ||||
BV_OBL = 3 | BV_OBL = 3 | ||||
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | CompressionStep.CLEAN_TMP: [ | ||||
"{out_dir}/{graph_name}-bv.graph", | "{out_dir}/{graph_name}-bv.graph", | ||||
"{out_dir}/{graph_name}-bv.obl", | "{out_dir}/{graph_name}-bv.obl", | ||||
"{out_dir}/{graph_name}-bv.offsets", | "{out_dir}/{graph_name}-bv.offsets", | ||||
"{tmp_dir}", | "{tmp_dir}", | ||||
], | ], | ||||
} | } | ||||
class StepOption(ParamType): | |||||
"""click type for specifying a compression step on the CLI | |||||
parse either individual steps, specified as step names or integers, or step | |||||
ranges | |||||
""" | |||||
name = "compression step" | |||||
def convert(self, value, param, ctx) -> Set[CompressionStep]: | |||||
steps: Set[CompressionStep] = set() | |||||
specs = value.split(",") | |||||
for spec in specs: | |||||
if "-" in spec: # step range | |||||
(raw_l, raw_r) = spec.split("-", maxsplit=1) | |||||
if raw_l == "": # no left endpoint | |||||
raw_l = COMP_SEQ[0].name | |||||
if raw_r == "": # no right endpoint | |||||
raw_r = COMP_SEQ[-1].name | |||||
l_step = self.convert(raw_l, param, ctx) | |||||
r_step = self.convert(raw_r, param, ctx) | |||||
if len(l_step) != 1 or len(r_step) != 1: | |||||
self.fail(f"invalid step specification: {value}, " f"see --help") | |||||
l_idx = l_step.pop() | |||||
r_idx = r_step.pop() | |||||
steps = steps.union( | |||||
set(map(CompressionStep, range(l_idx.value, r_idx.value + 1))) | |||||
) | |||||
else: # singleton step | |||||
try: | |||||
steps.add(CompressionStep(int(spec))) # integer step | |||||
except ValueError: | |||||
try: | |||||
steps.add(CompressionStep[spec.upper()]) # step name | |||||
except KeyError: | |||||
self.fail( | |||||
f"invalid step specification: {value}, " f"see --help" | |||||
) | |||||
return steps | |||||
def do_step(step, conf): | def do_step(step, conf): | ||||
cmd = " ".join(STEP_ARGV[step]).format(**conf) | cmd = " ".join(STEP_ARGV[step]).format(**conf) | ||||
cmd_env = os.environ.copy() | cmd_env = os.environ.copy() | ||||
cmd_env["JAVA_TOOL_OPTIONS"] = conf["java_tool_options"] | cmd_env["JAVA_TOOL_OPTIONS"] = conf["java_tool_options"] | ||||
cmd_env["CLASSPATH"] = conf["classpath"] | cmd_env["CLASSPATH"] = conf["classpath"] | ||||
logging.info(f"running: {cmd}") | logging.info(f"running: {cmd}") | ||||
▲ Show 20 Lines • Show All 81 Lines • Show Last 20 Lines |