diff --git a/Makefile.local b/Makefile.local index f1c93e5..5000628 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,98 +1,95 @@ # -*- makefile -*- FLAKE = flake8 BINDIR = bin SRCDIR = swh REPO_PATH=./swh-loader-git-testdata # add -v for example FLAG= -TESTDIR = ./swh/tests - DB=softwareheritage-dev -DB_TEST=$(DB)-test -SWH_LOADER=$(BINDIR)/swh-git-loader +SWH_LOADER=$(BINDIR)/swh-loader-git SWH_DB_MANAGER=$(BINDIR)/swh-db-manager SWH_BACK=$(BINDIR)/swh-backend SQL_FOLDER=../swh-storage/sql/ # could use cProfile PROFILE_TYPE=profile FOLLOW_LOG=-f # Adapt python-path to use other swh modules _PYPATH=`pwd`:`pwd`/../swh-core:`pwd`/../swh-storage deps: apt-get install -y \ python3 \ python3-pygit2 \ python3-psycopg2 \ python3-nose \ python3-flask \ python3-requests \ python3-retrying \ ipython3 cover: PYTHONPATH=$(_PYPATH) make coverage clean: - rm -rf /tmp/swh-git-loader/content-storage /tmp/swh-git-loader/test/ + rm -rf /tmp/swh-loader-git/content-storage /tmp/swh-loader-git/test/ prepare: - mkdir -p /tmp/swh-git-loader/content-storage /tmp/swh-git-loader/test/ + mkdir -p /tmp/swh-loader-git/content-storage /tmp/swh-loader-git/test/ cleandb: clean PYTHONPATH=$(_PYPATH) $(SWH_DB_MANAGER) $(FLAG) cleandb run-remote: PYTHONPATH=$(_PYPATH) $(SWH_LOADER) $(FLAG) --config ./resources/remote-git-loader.ini load $(REPO_PATH) run-local: PYTHONPATH=$(_PYPATH) $(SWH_LOADER) $(FLAG) --config ./resources/local-git-loader.ini load $(REPO_PATH) run: # works with the default ~/.config/swh/git-loader.ini file PYTHONPATH=$(_PYPATH) $(SWH_LOADER) $(FLAG) load $(REPO_PATH) run-back: PYTHONPATH=$(_PYPATH) $(SWH_BACK) $(FLAG) connect-db: psql -d $(DB) create-db: cd $(SQL_FOLDER) && make clean initdb drop-db: cd $(SQL_FOLDER) && make clean dropdb check-meta: @echo "Repository: $(REPO_PATH)" @echo "Git metadata:" @$(BINDIR)/dir-git-repo-meta.sh $(REPO_PATH) @echo @echo "DB metadata:" - @$(BINDIR)/db-git-repo-meta.sh $(DB) $(REPO_PATH) + @$(BINDIR)/db-git-repo-meta.sh $(DB) @echo log-loader: - tail $(FOLLOW_LOG) /tmp/swh-git-loader/log/sgloader.log + tail $(FOLLOW_LOG) /tmp/swh-loader-git/log/sgloader.log log-back: - tail $(FOLLOW_LOG) /tmp/swh-git-loader/log/back.log + tail $(FOLLOW_LOG) /tmp/swh-loader-git/log/back.log profile-run: PYTHONPATH=$(_PYPATH) python3 -m $(PROFILE_TYPE) -o ./scratch/swhgitloader.$(PROFILE_TYPE) ./scratch/profile-swhgitloader.py profile-stats: PYTHONPATH=$(_PYPATH) ./scratch/analyse-profile.py include Makefile.tests diff --git a/Makefile.tests b/Makefile.tests index 6febb29..0c1cb42 100644 --- a/Makefile.tests +++ b/Makefile.tests @@ -1,63 +1,74 @@ # -*- makefile -*- +NOSEFLAGS=--nologcapture -v +DB_TEST=$(DB)-test +TESTDIR = ./swh/tests test-connect-db: - psql -d $(DB_TEST) + psql $(DB_TEST) test-create-db: cd $(SQL_FOLDER) && make clean initdb DBNAME=$(DB_TEST) test-drop-db: cd $(SQL_FOLDER) && make clean dropdb DBNAME=$(DB_TEST) +test-cleandb: + PYTHONPATH=$(_PYPATH) $(SWH_DB_MANAGER) $(FLAG) --config ./resources/test/db-manager.ini cleandb + +test-check-meta: + @echo "DB $(DB_TEST) metadata:" + @$(BINDIR)/db-git-repo-meta.sh $(DB_TEST) + @echo + tests: PYTHONPATH=$(_PYPATH) make test test-run-back: PYTHONPATH=$(_PYPATH) $(SWH_BACK) $(FLAG) --config ./resources/test/back.ini test-http: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_http.py test-swhrepo: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_swhrepo.py test-api: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api*.py test-api-post-per-type: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_post_*.py test-api-content: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_content.py test-api-directory: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_directory.py test-api-revision: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_revision.py test-api-release: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_release.py test-api-occurrence: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_occurrence.py test-api-home: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_home.py test-api-origin: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_origin.py test-api-person: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_person.py test-file: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_file.py test-remote-loader: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_remote_loader.py test-local-loader: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_local_loader.py test-loaders: test-local-loader test-remote-loader diff --git a/resources/test/db-manager.ini b/resources/test/db-manager.ini new file mode 100644 index 0000000..679bc8c --- /dev/null +++ b/resources/test/db-manager.ini @@ -0,0 +1,7 @@ +[main] + +# Where to store the logs +log_dir = swh-loader-git/log + +# url access to db +db_url = dbname=softwareheritage-dev-test