Page MenuHomeSoftware Heritage
Paste P1121

bakfill script for sha1 based ranges
ActivePublic

Authored by vsellier on Aug 17 2021, 12:33 PM.
#!/bin/zsh
# From
setopt shwordsplit
objtype=extid
PARTITIONS=32
PARAMS="--dry-run"
mkdir -p logs/$objtype
for i in `seq 0 $PARTITIONS`; do
CMD=$(python3 backfill_command.py $PARTITIONS $i)
if [ -z "$CMD" ]; then
continue
fi
CMD="${CMD} ${PARAMS}"
echo "Starting ${objtype} $CMD"
echo " $CMD"
SWH_CONFIG_FILENAME=/etc/softwareheritage/journal/backfill.yml stdbuf -o0 $CMD 2>&1 | stdbuf -o0 tee >(gzip -c > logs/$objtype/$i.log.gz) &
done
wait