Changeset View
Changeset View
Standalone View
Standalone View
swh/graph/config.py
Show All 33 Lines | for path in try_paths: | ||||
return str(glob[0]) | return str(glob[0]) | ||||
raise RuntimeError("swh-graph JAR not found. Have you run `make java`?") | raise RuntimeError("swh-graph JAR not found. Have you run `make java`?") | ||||
def check_config(conf): | def check_config(conf): | ||||
"""check configuration and propagate defaults""" | """check configuration and propagate defaults""" | ||||
conf = conf.copy() | conf = conf.copy() | ||||
if "batch_size" not in conf: | if "batch_size" not in conf: | ||||
conf["batch_size"] = "1000000000" # 1 billion | # Use 0.1% of the RAM as a batch size: | ||||
# ~1 billion for big servers, ~10 million for small desktop machines | |||||
conf["batch_size"] = int(psutil.virtual_memory().total / 1000) | |||||
if "max_ram" not in conf: | if "max_ram" not in conf: | ||||
conf["max_ram"] = str(psutil.virtual_memory().total) | conf["max_ram"] = str(psutil.virtual_memory().total) | ||||
if "java_tool_options" not in conf: | if "java_tool_options" not in conf: | ||||
conf["java_tool_options"] = " ".join( | conf["java_tool_options"] = " ".join( | ||||
[ | [ | ||||
"-Xmx{max_ram}", | "-Xmx{max_ram}", | ||||
"-XX:PretenureSizeThreshold=512M", | "-XX:PretenureSizeThreshold=512M", | ||||
"-XX:MaxNewSize=4G", | "-XX:MaxNewSize=4G", | ||||
▲ Show 20 Lines • Show All 61 Lines • Show Last 20 Lines |