Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7124192
D3910.id.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
7 KB
Subscribers
None
D3910.id.diff
View Options
diff --git a/swh/graph/cli.py b/swh/graph/cli.py
--- a/swh/graph/cli.py
+++ b/swh/graph/cli.py
@@ -3,25 +3,65 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-import aiohttp
+# WARNING: do not import unnecessary things here to keep cli startup time under
+# control
import click
import logging
-import shutil
import sys
from pathlib import Path
-from typing import Any, Dict, Tuple
+from typing import Any, Dict, Tuple, Set, TYPE_CHECKING
-import swh.model.exceptions
-
-from swh.core import config
from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup
-from swh.graph import client, webgraph
-from swh.graph.backend import NODE2PID_EXT, PID2NODE_EXT
-from swh.graph.pid import PidToNodeMap, NodeToPidMap
-from swh.graph.server.app import make_app
-from swh.graph.backend import Backend
-from swh.model.identifiers import parse_persistent_identifier
+
+if TYPE_CHECKING:
+ from swh.graph.webgraph import CompressionStep # noqa
+
+
+class StepOption(click.ParamType):
+ """click type for specifying a compression step on the CLI
+
+ parse either individual steps, specified as step names or integers, or step
+ ranges
+
+ """
+
+ name = "compression step"
+
+ def convert(self, value, param, ctx): # type: (...) -> Set[CompressionStep]
+ from swh.graph.webgraph import CompressionStep, COMP_SEQ # noqa
+
+ steps: Set[CompressionStep] = set()
+
+ specs = value.split(",")
+ for spec in specs:
+ if "-" in spec: # step range
+ (raw_l, raw_r) = spec.split("-", maxsplit=1)
+ if raw_l == "": # no left endpoint
+ raw_l = COMP_SEQ[0].name
+ if raw_r == "": # no right endpoint
+ raw_r = COMP_SEQ[-1].name
+ l_step = self.convert(raw_l, param, ctx)
+ r_step = self.convert(raw_r, param, ctx)
+ if len(l_step) != 1 or len(r_step) != 1:
+ self.fail(f"invalid step specification: {value}, " f"see --help")
+ l_idx = l_step.pop()
+ r_idx = r_step.pop()
+ steps = steps.union(
+ set(map(CompressionStep, range(l_idx.value, r_idx.value + 1)))
+ )
+ else: # singleton step
+ try:
+ steps.add(CompressionStep(int(spec))) # integer step
+ except ValueError:
+ try:
+ steps.add(CompressionStep[spec.upper()]) # step name
+ except KeyError:
+ self.fail(
+ f"invalid step specification: {value}, " f"see --help"
+ )
+
+ return steps
class PathlibPath(click.Path):
@@ -45,8 +85,9 @@
@click.pass_context
def cli(ctx, config_file):
"""Software Heritage graph tools."""
- ctx.ensure_object(dict)
+ from swh.core import config
+ ctx.ensure_object(dict)
conf = config.read(config_file, DEFAULT_CONFIG)
if "graph" not in conf:
raise ValueError(
@@ -61,6 +102,8 @@
@click.pass_context
def api_client(ctx, host, port):
"""client for the graph REST service"""
+ from swh.graph import client
+
url = "http://{}:{}".format(host, port)
app = client.RemoteGraphClient(url)
@@ -76,11 +119,15 @@
def dump_pid2node(filename):
+ from swh.graph.pid import PidToNodeMap
+
for (pid, int) in PidToNodeMap(filename):
print("{}\t{}".format(pid, int))
def dump_node2pid(filename):
+ from swh.graph.pid import NodeToPidMap
+
for (int, pid) in NodeToPidMap(filename):
print("{}\t{}".format(int, pid))
@@ -90,6 +137,8 @@
filename
"""
+ from swh.graph.pid import PidToNodeMap
+
with open(filename, "wb") as dst:
for line in sys.stdin:
(str_pid, str_int) = line.split()
@@ -101,6 +150,8 @@
filename
"""
+ from swh.graph.pid import NodeToPidMap
+
node2pid = NodeToPidMap(filename, mode="wb", length=length)
for line in sys.stdin:
(str_int, str_pid) = line.split()
@@ -184,6 +235,8 @@
required by the chosen map type (by PID for pid2node, by int for node2pid)
"""
+ from swh.graph.pid import PidToNodeMap, NodeToPidMap
+
with open(filename, "wb") as f:
if map_type == "pid2node":
for line in sys.stdin:
@@ -215,6 +268,11 @@
readline()) in stdin will be preserved in stdout.
"""
+ import swh.model.exceptions
+ from swh.graph.backend import NODE2PID_EXT, PID2NODE_EXT
+ from swh.graph.pid import PidToNodeMap, NodeToPidMap
+ from swh.model.identifiers import parse_persistent_identifier
+
success = True # no identifiers failed to be looked up
pid2node = PidToNodeMap(f"{graph}.{PID2NODE_EXT}")
node2pid = NodeToPidMap(f"{graph}.{NODE2PID_EXT}")
@@ -278,6 +336,10 @@
@click.pass_context
def serve(ctx, host, port, graph):
"""run the graph REST service"""
+ import aiohttp
+ from swh.graph.server.app import make_app
+ from swh.graph.backend import Backend
+
backend = Backend(graph_path=graph, config=ctx.obj["config"])
app = make_app(backend=backend)
@@ -307,7 +369,7 @@
"--steps",
"-s",
metavar="STEPS",
- type=webgraph.StepOption(),
+ type=StepOption(),
help="run only these compression steps (default: all steps)",
)
@click.pass_context
@@ -325,6 +387,8 @@
also supported.
"""
+ from swh.graph import webgraph
+
graph_name = graph.name
in_dir = graph.parent
try:
@@ -360,6 +424,8 @@
The command outputs the path to the memory cache directory (particularly
useful when relying on the default value).
"""
+ import shutil
+
cache.mkdir(parents=True)
for src in Path(graph).parent.glob("*"):
dst = cache / src.name
diff --git a/swh/graph/webgraph.py b/swh/graph/webgraph.py
--- a/swh/graph/webgraph.py
+++ b/swh/graph/webgraph.py
@@ -15,7 +15,6 @@
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Set
-from click import ParamType
from swh.graph.config import check_config_compress
@@ -136,50 +135,6 @@
}
-class StepOption(ParamType):
- """click type for specifying a compression step on the CLI
-
- parse either individual steps, specified as step names or integers, or step
- ranges
-
- """
-
- name = "compression step"
-
- def convert(self, value, param, ctx) -> Set[CompressionStep]:
- steps: Set[CompressionStep] = set()
-
- specs = value.split(",")
- for spec in specs:
- if "-" in spec: # step range
- (raw_l, raw_r) = spec.split("-", maxsplit=1)
- if raw_l == "": # no left endpoint
- raw_l = COMP_SEQ[0].name
- if raw_r == "": # no right endpoint
- raw_r = COMP_SEQ[-1].name
- l_step = self.convert(raw_l, param, ctx)
- r_step = self.convert(raw_r, param, ctx)
- if len(l_step) != 1 or len(r_step) != 1:
- self.fail(f"invalid step specification: {value}, " f"see --help")
- l_idx = l_step.pop()
- r_idx = r_step.pop()
- steps = steps.union(
- set(map(CompressionStep, range(l_idx.value, r_idx.value + 1)))
- )
- else: # singleton step
- try:
- steps.add(CompressionStep(int(spec))) # integer step
- except ValueError:
- try:
- steps.add(CompressionStep[spec.upper()]) # step name
- except KeyError:
- self.fail(
- f"invalid step specification: {value}, " f"see --help"
- )
-
- return steps
-
-
def do_step(step, conf):
cmd = " ".join(STEP_ARGV[step]).format(**conf)
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Dec 20 2024, 11:23 PM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3217491
Attached To
D3910: cli: speedup the `swh` cli command startup time
Event Timeline
Log In to Comment