restart a backfill where it has stopped previously
ActivePublic
Actions

Authored by vsellier on Aug 18 2021, 2:37 PM.

Tags

None

Subscribers

None

	#!/bin/zsh
	#set -x
	setopt shwordsplit

	cat >/tmp/script.py <<EOF
	import sys
	n=int(sys.argv[1])
	print(format(n, "06x"))
	EOF

	inc=524288 #080000
	objtype=extid

	mkdir -p logs/$objtype

	compute_end() {
	i=$1
	ed=$(( inc * ($i +1) ))
	eh=$(python3 /tmp/script.py $ed)
	echo $eh
	}

	get_start() {
	i=$1
	zgrep Processing logs.1/$i.log.gz \| tail -n1 \| awk '{print $7}'
	}

	launch_backfill() {
	i=$1
	start=$2
	end=$3

	stop_param=""

	if [ -n "$end" ]; then
	stop_param="--end-object $end"
	fi

	CMD="swh --log-config /etc/softwareheritage/journal/backfill_logger.yml storage backfill ${objtype} --start-object $start $stop_param"
	echo $CMD
	SWH_CONFIG_FILENAME=/etc/softwareheritage/journal/backfill.yml stdbuf -o0 $CMD 2>&1 \| stdbuf -o0 tee >(gzip -c > logs/$objtype/$i.log.gz) &
	}

	for i in $(seq 0 30); do
	#echo -n "$i: start object:"
	start="$(get_start $i)"
	#echo -n $start
	end=$(compute_end $i)
	#echo " end_object: $end:"
	launch_backfill $i $start $end
	done

	start=$(get_start 31)
	#echo $start
	launch_backfill 31 $start

	wait

restart a backfill where it has stopped previouslyActivePublicActions