No OneTemporary
Actions

Size

9 KB

Subscribers

None

View Options

	diff --git a/tests/run_tests.sh b/tests/run_tests.sh
	index 64feecc..0237089 100755
	--- a/tests/run_tests.sh
	+++ b/tests/run_tests.sh
	@@ -1,177 +1,179 @@
	#!/bin/bash

	# Main script to run high level tests on the Software Heritage stack

	# Use a temporary directory as working directory
	WORKDIR=/tmp/swh-docker-dev_tests
	# Create it if it does not exist
	mkdir $WORKDIR 2>/dev/null
	# Ensure it is empty before running the tests
	rm -rf $WORKDIR/*

	# We want the script to exit at the first encountered error
	set -e

	# Get test scripts directory
	TEST_SCRIPTS_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)

	# Set the docker-compose.yml file to use
	export COMPOSE_FILE=$TEST_SCRIPTS_DIR/../docker-compose.yml

	# Useful global variables
	SWH_WEB_API_BASEURL="http://localhost:5004/api/1"
	CURRENT_TEST_SCRIPT=""

	# Colored output related variables and functions (only if stdout is a terminal)
	if test -t 1; then
	GREEN='\033[0;32m'
	RED='\033[0;31m'
	NC='\033[0m'
	+else
	+ DOCO_OPTIONS='--no-ansi'
	fi

	# Remove previously dumped service logs file if any
	rm -f $TEST_SCRIPTS_DIR/swh-docker-compose.logs

	function colored_output {
	local msg="$2"
	if [ "$CURRENT_TEST_SCRIPT" != "" ]; then
	msg="[$CURRENT_TEST_SCRIPT] $msg"
	fi
	echo -e "${1}${msg}${NC}"
	}

	function status_message {
	colored_output ${GREEN} "$1"
	}

	function error_message {
	colored_output ${RED} "$1"
	}

	function dump_docker_logs {
	error_message "Dumping logs for all services in file $TEST_SCRIPTS_DIR/swh-docker-compose.logs"
	docker-compose logs > $TEST_SCRIPTS_DIR/swh-docker-compose.logs
	}

	# Exit handler that will get called when this script terminates
	function finish {
	if [ $? -ne 0 ] && [ "$CURRENT_TEST_SCRIPT" != "" ]; then
	local SCRIPT_NAME=$CURRENT_TEST_SCRIPT
	CURRENT_TEST_SCRIPT=""
	error_message "An error occurred when running test script ${SCRIPT_NAME}"
	dump_docker_logs
	fi
	- docker-compose down
	+ docker-compose $DOCO_OPTIONS down
	rm -rf $WORKDIR
	}
	trap finish EXIT

	# Docker-compose events listener that will be executed in background
	# Parameters:
	# $1: PID of parent process
	function listen_docker_events {
	- docker-compose events \| while read event
	+ docker-compose $DOCO_OPTIONS events \| while read event
	do
	service=$(echo $event \| cut -d " " -f7 \| sed 's/^name=swh-docker-dev_$.*$_1)/\1/')
	event_type=$(echo $event \| cut -d ' ' -f4)
	# "docker-compose down" has been called, exiting this child process
	if [ "$event_type" = "kill" ] ; then
	exit
	# a swh service crashed, sending signal to parent process to exit with error
	elif [ "$event_type" = "die" ] ; then
	if [[ "$service" =~ ^swh.* ]]; then
	error_message "Service $service died unexpectedly, exiting"
	dump_docker_logs
	kill -s SIGUSR1 $1; exit
	fi
	fi
	done
	}
	trap "exit 1" SIGUSR1

	declare -A SERVICE_LOGS_NB_LINES_READ

	# Function to wait for a specific string to be outputted in a specific
	# docker-compose service logs.
	# When called multiple times on the same service, only the newly outputted
	# logs since the last call will be processed.
	# Parameters:
	# $1: a timeout value in seconds to stop waiting and exit with error
	# $2: docker-compose service name
	# $3: the string to look for in the produced logs
	function wait_for_service_output {
	local nb_lines_to_skip=0
	if [[ -v "SERVICE_LOGS_NB_LINES_READ[$2]" ]]; then
	let nb_lines_to_skip=${SERVICE_LOGS_NB_LINES_READ[$2]}+1
	fi
	SECONDS=0
	- local service_logs=$(docker-compose logs $2 \| tail -n +$nb_lines_to_skip)
	+ local service_logs=$(docker-compose $DOCO_OPTIONS logs $2 \| tail -n +$nb_lines_to_skip)
	until echo -ne "$service_logs" \| grep -m 1 "$3" >/dev/null ; do
	sleep 1;
	if (( $SECONDS > $1 )); then
	error_message "Could not find pattern \"$3\" in $2 service logs after $1 seconds"
	exit 1
	fi
	let nb_lines_to_skip+=$(echo -ne "$service_logs" \| wc -l)
	- service_logs=$(docker-compose logs $2 \| tail -n +$nb_lines_to_skip)
	+ service_logs=$(docker-compose $DOCO_OPTIONS logs $2 \| tail -n +$nb_lines_to_skip)
	done
	let nb_lines_to_skip+=$(echo -ne "$service_logs" \| wc -l)
	SERVICE_LOGS_NB_LINES_READ[$2]=$nb_lines_to_skip
	}

	# Function to make an HTTP request and gets its response.
	# It should be used the following way:
	# response=$(http_request <method> <url>)
	# Parameters:
	# $1: http method name (GET, POST, ...)
	# $2: request url
	function http_request {
	local response=$(curl -sS -X $1 $2)
	echo $response
	}

	# Function to check that an HTTP request ends up with no errors.
	# If the HTTP response code is different from 200, an error will
	# be raised and the main script will terminate
	# Parameters:
	# $1: http method name (GET, POST, ...)
	# $2: request url
	function http_request_check {
	curl -sSf -X $1 $2 > /dev/null
	}

	# Function to run the content of a script dedicated to test a specific
	# part of the Software Heritage stack.
	function run_test_script {
	local SCRIPT_NAME=$(basename $1)
	status_message "Executing test script $SCRIPT_NAME"
	CURRENT_TEST_SCRIPT=$SCRIPT_NAME
	source $1
	}

	# Move to work directory
	cd $WORKDIR

	# Start the docker-compose event handler as a background process
	status_message "Starting docker-compose events listener"
	listen_docker_events $$ &

	# Start the docker-compose environment including the full Software Heritage stack
	status_message "Starting swh docker-compose environment"
	-docker-compose up -d
	+docker-compose $DOCO_OPTIONS up -d

	# Ensure all swh services are up before running tests
	status_message "Waiting for swh services to be up"
	-docker-compose exec -T swh-storage wait-for-it localhost:5002 -s --timeout=0
	-docker-compose exec -T swh-objstorage wait-for-it localhost:5003 -s --timeout=0
	-docker-compose exec -T swh-web wait-for-it localhost:5004 -s --timeout=0
	-docker-compose exec -T swh-vault-api wait-for-it localhost:5005 -s --timeout=0
	-docker-compose exec -T swh-deposit wait-for-it localhost:5006 -s --timeout=0
	-docker-compose exec -T swh-idx-storage wait-for-it localhost:5007 -s --timeout=0
	-docker-compose exec -T swh-scheduler-api wait-for-it localhost:5008 -s --timeout=0
	+docker-compose $DOCO_OPTIONS exec -T swh-storage wait-for-it localhost:5002 -s --timeout=0
	+docker-compose $DOCO_OPTIONS exec -T swh-objstorage wait-for-it localhost:5003 -s --timeout=0
	+docker-compose $DOCO_OPTIONS exec -T swh-web wait-for-it localhost:5004 -s --timeout=0
	+docker-compose $DOCO_OPTIONS exec -T swh-vault-api wait-for-it localhost:5005 -s --timeout=0
	+docker-compose $DOCO_OPTIONS exec -T swh-deposit wait-for-it localhost:5006 -s --timeout=0
	+docker-compose $DOCO_OPTIONS exec -T swh-idx-storage wait-for-it localhost:5007 -s --timeout=0
	+docker-compose $DOCO_OPTIONS exec -T swh-scheduler-api wait-for-it localhost:5008 -s --timeout=0

	# Execute test scripts
	for test_script in $TEST_SCRIPTS_DIR/test_*; do
	run_test_script ${test_script}
	CURRENT_TEST_SCRIPT=""
	done
	diff --git a/tests/test_01_loader_git.sh b/tests/test_01_loader_git.sh
	index ac8f44c..7342050 100644
	--- a/tests/test_01_loader_git.sh
	+++ b/tests/test_01_loader_git.sh
	@@ -1,70 +1,70 @@
	#!/bin/bash
	shopt -s nullglob extglob

	TEST_GIT_REPO_NAME="swh-loader-tar"
	TEST_GIT_REPO_URL="https://forge.softwareheritage.org/source/${TEST_GIT_REPO_NAME}.git"

	status_message "Scheduling the loading of the git repository located at ${TEST_GIT_REPO_URL}"

	-docker-compose exec -T swh-scheduler-api swh-scheduler task add origin-update-git repo_url=$TEST_GIT_REPO_URL
	+docker-compose $DOCO_OPTIONS exec -T swh-scheduler-api swh-scheduler task add origin-update-git repo_url=$TEST_GIT_REPO_URL

	status_message "Waiting for the git loading task to complete"

	wait_for_service_output 300 swh-loader "swh.loader.git.tasks.UpdateGitRepository.*succeeded"

	status_message "The loading task has been successfully executed"

	status_message "Getting all git objects contained in the repository"
	git clone $TEST_GIT_REPO_URL
	cd $TEST_GIT_REPO_NAME
	cd "$(git rev-parse --git-path objects)"
	for p in pack/pack-*([0-9a-f]).idx ; do
	git show-index < $p \| cut -f 2 -d ' ' > $WORKDIR/git_objects
	done
	for o in [0-9a-f][0-9a-f]/*([0-9a-f]) ; do
	echo ${o/\/} >> $WORKDIR/git_objects
	done

	declare -ga CONTENTS
	declare -ga DIRECTORIES
	declare -ga REVISIONS
	declare -ga RELEASES

	while IFS='' read -r object \|\| [[ -n "$object" ]]; do
	object_type=$(git cat-file -t $object)
	if [ "$object_type" = "blob" ]; then
	CONTENTS+=($object)
	elif [ "$object_type" = "tree" ]; then
	DIRECTORIES+=($object)
	elif [ "$object_type" = "commit" ]; then
	REVISIONS+=($object)
	elif [ "$object_type" = "tag" ]; then
	RELEASES+=($object)
	fi
	done < $WORKDIR/git_objects

	status_message "Checking all git objects have been successfully loaded into the archive"

	status_message "Checking contents"
	for content in "${CONTENTS[@]}"; do
	http_request_check GET ${SWH_WEB_API_BASEURL}/content/sha1_git:$content/
	done
	status_message "All contents have been successfully loaded into the archive"

	status_message "Checking directories"
	for directory in "${DIRECTORIES[@]}"; do
	http_request_check GET ${SWH_WEB_API_BASEURL}/directory/$directory/
	done
	status_message "All directories have been successfully loaded into the archive"

	status_message "Checking revisions"
	for revision in "${REVISIONS[@]}"; do
	http_request_check GET ${SWH_WEB_API_BASEURL}/revision/$revision/
	done
	status_message "All revisions have been successfully loaded into the archive"

	status_message "Checking releases"
	for release in "${RELEASES[@]}"; do
	http_request_check GET ${SWH_WEB_API_BASEURL}/release/$release/
	done
	status_message "All releases have been successfully loaded into the archive"

File Metadata

Mime Type: text/x-diff
Expires: Mon, Aug 18, 11:04 PM (1 w, 4 h ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3263793

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions