Page MenuHomeSoftware Heritage
Paste P1124

restart a backfill where it has stopped previously
ActivePublic

Authored by vsellier on Aug 18 2021, 2:37 PM.
#!/bin/zsh
#set -x
setopt shwordsplit
cat >/tmp/script.py <<EOF
import sys
n=int(sys.argv[1])
print(format(n, "06x"))
EOF
inc=524288 #080000
objtype=extid
mkdir -p logs/$objtype
compute_end() {
i=$1
ed=$(( inc * ($i +1) ))
eh=$(python3 /tmp/script.py $ed)
echo $eh
}
get_start() {
i=$1
zgrep Processing logs.1/$i.log.gz | tail -n1 | awk '{print $7}'
}
launch_backfill() {
i=$1
start=$2
end=$3
stop_param=""
if [ -n "$end" ]; then
stop_param="--end-object $end"
fi
CMD="swh --log-config /etc/softwareheritage/journal/backfill_logger.yml storage backfill ${objtype} --start-object $start $stop_param"
echo $CMD
SWH_CONFIG_FILENAME=/etc/softwareheritage/journal/backfill.yml stdbuf -o0 $CMD 2>&1 | stdbuf -o0 tee >(gzip -c > logs/$objtype/$i.log.gz) &
}
for i in $(seq 0 30); do
#echo -n "$i: start object:"
start="$(get_start $i)"
#echo -n $start
end=$(compute_end $i)
#echo " end_object: $end:"
launch_backfill $i $start $end
done
start=$(get_start 31)
#echo $start
launch_backfill 31 $start
wait