Page MenuHomeSoftware Heritage
Paste P927

backfill script with the right backfill.yml and logging.yml configuration files
ActivePublic

Authored by vsellier on Jan 20 2021, 6:45 PM.
#!/bin/bash
objtype=origin_visit_status
first=0
mkdir -p logs/$objtype
inc=50000
# 458591
for i in `seq 0 10`; do
prefix=$(( first + i * $inc))
if [ $prefix -eq $first ]; then
start=$first
else
start=$prefix
fi
end=$((prefix + $inc));
echo "Starting ${objtype} backfill for range $start -> $end"
swh --log-config logging.yml storage --config-file backfill.yml backfill --start-object $start --end-object $end $objtype 2>&1 | tee -a >(gzip -c > logs/$objtype/$prefix.log.gz) &
done
wait