diff --git a/conftest.py b/conftest.py deleted file mode 100644 --- a/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (C) 2020 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import pytest - -pytest_plugins = [ - "swh.scheduler.pytest_plugin", - "swh.storage.pytest_plugin", - "swh.core.pytest_plugin", -] - - -@pytest.fixture(scope="session") -def swh_scheduler_celery_includes(swh_scheduler_celery_includes): - return swh_scheduler_celery_includes + [ - "swh.deposit.loader.tasks", - ] diff --git a/requirements-server.txt b/requirements-server.txt --- a/requirements-server.txt +++ b/requirements-server.txt @@ -1,4 +1,3 @@ django >= 2, < 4 djangorestframework -psycopg2 < 2.9 setuptools diff --git a/requirements-test.txt b/requirements-test-server.txt copy from requirements-test.txt copy to requirements-test-server.txt --- a/requirements-test.txt +++ b/requirements-test-server.txt @@ -1,13 +1,7 @@ -pytest < 7.0.0 # v7.0.0 removed _pytest.tmpdir.TempdirFactory, which is used by some of the pytest plugins we use pytest-django -pytest-mock swh.scheduler[testing] swh.loader.core[testing] pytest-postgresql >=3, < 4.0.0 # version 4.0 depends on psycopg 3. https://github.com/ClearcodeHQ/pytest-postgresql/blob/main/CHANGES.rst#400 -requests_mock django-stubs djangorestframework-stubs >= 1.4 django-test-migrations - -types-requests -types-pyyaml diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,13 +1,6 @@ pytest < 7.0.0 # v7.0.0 removed _pytest.tmpdir.TempdirFactory, which is used by some of the pytest plugins we use -pytest-django pytest-mock -swh.scheduler[testing] -swh.loader.core[testing] -pytest-postgresql >=3, < 4.0.0 # version 4.0 depends on psycopg 3. https://github.com/ClearcodeHQ/pytest-postgresql/blob/main/CHANGES.rst#400 requests_mock -django-stubs -djangorestframework-stubs >= 1.4 -django-test-migrations types-requests types-pyyaml diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -51,7 +51,8 @@ setup_requires=["setuptools-scm"], use_scm_version=True, extras_require={ - "testing": parse_requirements("test", "server", "swh-server"), + "testing": parse_requirements("test", "test-server", "server", "swh-server"), + "testing-core": parse_requirements("test"), "server": parse_requirements("server", "swh-server"), }, include_package_data=True, diff --git a/swh/deposit/api/collection.py b/swh/deposit/api/collection.py --- a/swh/deposit/api/collection.py +++ b/swh/deposit/api/collection.py @@ -9,7 +9,7 @@ from rest_framework import status from rest_framework.generics import ListAPIView -from ..config import DEPOSIT_STATUS_LOAD_SUCCESS, EDIT_IRI +from ..constants import DEPOSIT_STATUS_LOAD_SUCCESS, EDIT_IRI from ..models import Deposit from ..parsers import ( SWHAtomEntryParser, diff --git a/swh/deposit/api/common.py b/swh/deposit/api/common.py --- a/swh/deposit/api/common.py +++ b/swh/deposit/api/common.py @@ -44,7 +44,8 @@ ) from swh.scheduler.utils import create_oneshot_task_dict -from ..config import ( +from ..config import APIConfig +from ..constants import ( ARCHIVE_KEY, ARCHIVE_TYPE, CONT_FILE_IRI, @@ -58,7 +59,6 @@ RAW_METADATA_KEY, SE_IRI, STATE_IRI, - APIConfig, ) from ..errors import ( BAD_REQUEST, diff --git a/swh/deposit/api/edit.py b/swh/deposit/api/edit.py --- a/swh/deposit/api/edit.py +++ b/swh/deposit/api/edit.py @@ -8,7 +8,7 @@ from swh.deposit.models import Deposit from swh.model.swhids import QualifiedSWHID -from ..config import DEPOSIT_STATUS_LOAD_SUCCESS +from ..constants import DEPOSIT_STATUS_LOAD_SUCCESS from ..errors import BAD_REQUEST, DepositError, ParserError from ..parsers import SWHAtomEntryParser, SWHMultiPartParser from .common import APIDelete, APIPut, ParsedRequestHeaders diff --git a/swh/deposit/api/edit_media.py b/swh/deposit/api/edit_media.py --- a/swh/deposit/api/edit_media.py +++ b/swh/deposit/api/edit_media.py @@ -7,7 +7,7 @@ from rest_framework import status -from ..config import CONT_FILE_IRI +from ..constants import CONT_FILE_IRI from ..errors import BAD_REQUEST, DepositError from ..models import Deposit from ..parsers import SWHFileUploadTarParser, SWHFileUploadZipParser diff --git a/swh/deposit/api/private/__init__.py b/swh/deposit/api/private/__init__.py --- a/swh/deposit/api/private/__init__.py +++ b/swh/deposit/api/private/__init__.py @@ -8,7 +8,8 @@ from rest_framework.permissions import AllowAny from rest_framework.views import APIView -from ...config import METADATA_TYPE, APIConfig +from ...config import APIConfig +from ...constants import METADATA_TYPE from ...models import Deposit, DepositRequest diff --git a/swh/deposit/api/private/deposit_check.py b/swh/deposit/api/private/deposit_check.py --- a/swh/deposit/api/private/deposit_check.py +++ b/swh/deposit/api/private/deposit_check.py @@ -17,7 +17,7 @@ from swh.scheduler.utils import create_oneshot_task_dict from . import APIPrivateView, DepositReadMixin -from ...config import ARCHIVE_TYPE, DEPOSIT_STATUS_REJECTED, DEPOSIT_STATUS_VERIFIED +from ...constants import ARCHIVE_TYPE, DEPOSIT_STATUS_REJECTED, DEPOSIT_STATUS_VERIFIED from ...models import Deposit, DepositRequest from ..checks import check_metadata from ..common import APIGet diff --git a/swh/deposit/api/private/deposit_read.py b/swh/deposit/api/private/deposit_read.py --- a/swh/deposit/api/private/deposit_read.py +++ b/swh/deposit/api/private/deposit_read.py @@ -19,7 +19,8 @@ from swh.model.swhids import CoreSWHID from . import APIPrivateView, DepositReadMixin -from ...config import ARCHIVE_TYPE, SWH_PERSON +from ...config import SWH_PERSON +from ...constants import ARCHIVE_TYPE from ...models import Deposit from ...utils import parse_xml from ..common import APIGet diff --git a/swh/deposit/api/private/urls.py b/swh/deposit/api/private/urls.py --- a/swh/deposit/api/private/urls.py +++ b/swh/deposit/api/private/urls.py @@ -5,7 +5,7 @@ from django.conf.urls import url -from ...config import ( +from ...constants import ( PRIVATE_CHECK_DEPOSIT, PRIVATE_GET_DEPOSIT_METADATA, PRIVATE_GET_RAW_CONTENT, diff --git a/swh/deposit/api/service_document.py b/swh/deposit/api/service_document.py --- a/swh/deposit/api/service_document.py +++ b/swh/deposit/api/service_document.py @@ -11,7 +11,7 @@ ACCEPT_PACKAGINGS, APIBase, ) -from swh.deposit.config import COL_IRI +from swh.deposit.constants import COL_IRI from swh.deposit.models import DepositClient, DepositCollection diff --git a/swh/deposit/api/sword_edit.py b/swh/deposit/api/sword_edit.py --- a/swh/deposit/api/sword_edit.py +++ b/swh/deposit/api/sword_edit.py @@ -10,7 +10,7 @@ from swh.storage import get_storage from swh.storage.interface import StorageInterface -from ..config import EDIT_IRI, EM_IRI +from ..constants import EDIT_IRI, EM_IRI from ..models import Deposit from ..parsers import SWHAtomEntryParser, SWHMultiPartParser from .common import APIPost, ParsedRequestHeaders, Receipt diff --git a/swh/deposit/api/urls.py b/swh/deposit/api/urls.py --- a/swh/deposit/api/urls.py +++ b/swh/deposit/api/urls.py @@ -10,7 +10,15 @@ from django.conf.urls import url from django.shortcuts import render -from ..config import COL_IRI, CONT_FILE_IRI, EDIT_IRI, EM_IRI, SD_IRI, SE_IRI, STATE_IRI +from ..constants import ( + COL_IRI, + CONT_FILE_IRI, + EDIT_IRI, + EM_IRI, + SD_IRI, + SE_IRI, + STATE_IRI, +) from .collection import CollectionAPI from .content import ContentAPI from .edit import EditAPI diff --git a/swh/deposit/cli/admin.py b/swh/deposit/cli/admin.py --- a/swh/deposit/cli/admin.py +++ b/swh/deposit/cli/admin.py @@ -236,11 +236,11 @@ # to avoid loading too early django namespaces from datetime import datetime - from swh.deposit.config import ( + from swh.deposit.config import APIConfig + from swh.deposit.constants import ( DEPOSIT_STATUS_LOAD_FAILURE, DEPOSIT_STATUS_LOAD_SUCCESS, DEPOSIT_STATUS_VERIFIED, - APIConfig, ) from swh.deposit.models import Deposit diff --git a/swh/deposit/constants.py b/swh/deposit/constants.py new file mode 100644 --- /dev/null +++ b/swh/deposit/constants.py @@ -0,0 +1,35 @@ +# Copyright (C) 2017-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +# IRIs (Internationalized Resource identifier) sword 2.0 specified +EDIT_IRI = "edit_iri" +SE_IRI = "se_iri" +EM_IRI = "em_iri" +CONT_FILE_IRI = "cont_file_iri" +SD_IRI = "servicedocument" +COL_IRI = "upload" +STATE_IRI = "state_iri" +PRIVATE_GET_RAW_CONTENT = "private-download" +PRIVATE_CHECK_DEPOSIT = "check-deposit" +PRIVATE_PUT_DEPOSIT = "private-update" +PRIVATE_GET_DEPOSIT_METADATA = "private-read" +PRIVATE_LIST_DEPOSITS = "private-deposit-list" + +ARCHIVE_KEY = "archive" +METADATA_KEY = "metadata" +RAW_METADATA_KEY = "raw-metadata" + +ARCHIVE_TYPE = "archive" +METADATA_TYPE = "metadata" + +AUTHORIZED_PLATFORMS = ["development", "production", "testing"] + +DEPOSIT_STATUS_REJECTED = "rejected" +DEPOSIT_STATUS_PARTIAL = "partial" +DEPOSIT_STATUS_DEPOSITED = "deposited" +DEPOSIT_STATUS_VERIFIED = "verified" +DEPOSIT_STATUS_LOAD_SUCCESS = "done" +DEPOSIT_STATUS_LOAD_FAILURE = "failed" diff --git a/swh/deposit/migrations/0018_migrate_swhids.py b/swh/deposit/migrations/0018_migrate_swhids.py --- a/swh/deposit/migrations/0018_migrate_swhids.py +++ b/swh/deposit/migrations/0018_migrate_swhids.py @@ -13,7 +13,7 @@ from django.db import migrations from swh.core import config -from swh.deposit.config import DEPOSIT_STATUS_LOAD_SUCCESS +from swh.deposit.constants import DEPOSIT_STATUS_LOAD_SUCCESS from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.swhids import CoreSWHID, ObjectType, QualifiedSWHID from swh.storage import get_storage as get_storage_client diff --git a/swh/deposit/models.py b/swh/deposit/models.py --- a/swh/deposit/models.py +++ b/swh/deposit/models.py @@ -17,7 +17,7 @@ from swh.auth.django.models import OIDCUser -from .config import ( +from .constants import ( ARCHIVE_TYPE, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_LOAD_FAILURE, diff --git a/swh/deposit/parsers.py b/swh/deposit/parsers.py --- a/swh/deposit/parsers.py +++ b/swh/deposit/parsers.py @@ -11,7 +11,14 @@ import logging from xml.parsers.expat import ExpatError -from django.conf import settings +try: + from django.conf import settings +except ImportError: + # client-only install + class settings: # type: ignore + DEFAULT_CHARSET = "utf8" + + from rest_framework.parsers import BaseParser, FileUploadParser, MultiPartParser from swh.deposit.errors import ParserError diff --git a/swh/deposit/tests/api/conftest.py b/swh/deposit/tests/api/conftest.py --- a/swh/deposit/tests/api/conftest.py +++ b/swh/deposit/tests/api/conftest.py @@ -10,7 +10,7 @@ import pytest from swh.deposit.api.private.deposit_check import APIChecks -from swh.deposit.config import ( +from swh.deposit.constants import ( COL_IRI, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_VERIFIED, diff --git a/swh/deposit/tests/api/test_basic_auth.py b/swh/deposit/tests/api/test_basic_auth.py --- a/swh/deposit/tests/api/test_basic_auth.py +++ b/swh/deposit/tests/api/test_basic_auth.py @@ -10,7 +10,7 @@ from django.urls import reverse_lazy as reverse import pytest -from swh.deposit.config import SD_IRI +from swh.deposit.constants import SD_IRI from .test_service_document import check_response diff --git a/swh/deposit/tests/api/test_collection.py b/swh/deposit/tests/api/test_collection.py --- a/swh/deposit/tests/api/test_collection.py +++ b/swh/deposit/tests/api/test_collection.py @@ -9,7 +9,7 @@ from django.urls import reverse_lazy as reverse from rest_framework import status -from swh.deposit.config import COL_IRI, DEPOSIT_STATUS_REJECTED +from swh.deposit.constants import COL_IRI, DEPOSIT_STATUS_REJECTED from swh.deposit.parsers import parse_xml diff --git a/swh/deposit/tests/api/test_collection_add_to_origin.py b/swh/deposit/tests/api/test_collection_add_to_origin.py --- a/swh/deposit/tests/api/test_collection_add_to_origin.py +++ b/swh/deposit/tests/api/test_collection_add_to_origin.py @@ -8,7 +8,7 @@ from django.urls import reverse_lazy as reverse from rest_framework import status -from swh.deposit.config import COL_IRI, DEPOSIT_STATUS_LOAD_SUCCESS +from swh.deposit.constants import COL_IRI, DEPOSIT_STATUS_LOAD_SUCCESS from swh.deposit.models import Deposit from swh.deposit.parsers import parse_xml from swh.deposit.tests.common import post_atom diff --git a/swh/deposit/tests/api/test_collection_list.py b/swh/deposit/tests/api/test_collection_list.py --- a/swh/deposit/tests/api/test_collection_list.py +++ b/swh/deposit/tests/api/test_collection_list.py @@ -9,7 +9,11 @@ from requests.utils import parse_header_links from rest_framework import status -from swh.deposit.config import COL_IRI, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_PARTIAL +from swh.deposit.constants import ( + COL_IRI, + DEPOSIT_STATUS_DEPOSITED, + DEPOSIT_STATUS_PARTIAL, +) from swh.deposit.models import DepositCollection from swh.deposit.parsers import parse_xml diff --git a/swh/deposit/tests/api/test_collection_post_atom.py b/swh/deposit/tests/api/test_collection_post_atom.py --- a/swh/deposit/tests/api/test_collection_post_atom.py +++ b/swh/deposit/tests/api/test_collection_post_atom.py @@ -15,11 +15,11 @@ import pytest from rest_framework import status -from swh.deposit.config import ( +from swh.deposit.config import APIConfig +from swh.deposit.constants import ( COL_IRI, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_LOAD_SUCCESS, - APIConfig, ) from swh.deposit.models import Deposit, DepositCollection, DepositRequest from swh.deposit.parsers import parse_xml diff --git a/swh/deposit/tests/api/test_collection_post_binary.py b/swh/deposit/tests/api/test_collection_post_binary.py --- a/swh/deposit/tests/api/test_collection_post_binary.py +++ b/swh/deposit/tests/api/test_collection_post_binary.py @@ -12,7 +12,7 @@ import pytest from rest_framework import status -from swh.deposit.config import COL_IRI, DEPOSIT_STATUS_DEPOSITED +from swh.deposit.constants import COL_IRI, DEPOSIT_STATUS_DEPOSITED from swh.deposit.models import Deposit, DepositRequest from swh.deposit.parsers import parse_xml from swh.deposit.tests.common import ( diff --git a/swh/deposit/tests/api/test_collection_post_multipart.py b/swh/deposit/tests/api/test_collection_post_multipart.py --- a/swh/deposit/tests/api/test_collection_post_multipart.py +++ b/swh/deposit/tests/api/test_collection_post_multipart.py @@ -13,7 +13,7 @@ import pytest from rest_framework import status -from swh.deposit.config import COL_IRI, DEPOSIT_STATUS_DEPOSITED +from swh.deposit.constants import COL_IRI, DEPOSIT_STATUS_DEPOSITED from swh.deposit.models import Deposit, DepositRequest from swh.deposit.parsers import parse_xml from swh.deposit.tests.common import check_archive, post_multipart diff --git a/swh/deposit/tests/api/test_collection_reuse_slug.py b/swh/deposit/tests/api/test_collection_reuse_slug.py --- a/swh/deposit/tests/api/test_collection_reuse_slug.py +++ b/swh/deposit/tests/api/test_collection_reuse_slug.py @@ -9,7 +9,7 @@ from rest_framework import status import xmltodict -from swh.deposit.config import ( +from swh.deposit.constants import ( COL_IRI, DEPOSIT_STATUS_LOAD_FAILURE, DEPOSIT_STATUS_LOAD_SUCCESS, diff --git a/swh/deposit/tests/api/test_delete.py b/swh/deposit/tests/api/test_delete.py --- a/swh/deposit/tests/api/test_delete.py +++ b/swh/deposit/tests/api/test_delete.py @@ -10,7 +10,7 @@ from rest_framework import status import xmltodict -from swh.deposit.config import ( +from swh.deposit.constants import ( ARCHIVE_KEY, DEPOSIT_STATUS_DEPOSITED, EDIT_IRI, diff --git a/swh/deposit/tests/api/test_deposit_private_check.py b/swh/deposit/tests/api/test_deposit_private_check.py --- a/swh/deposit/tests/api/test_deposit_private_check.py +++ b/swh/deposit/tests/api/test_deposit_private_check.py @@ -13,7 +13,7 @@ MANDATORY_ARCHIVE_MISSING, MANDATORY_ARCHIVE_UNSUPPORTED, ) -from swh.deposit.config import ( +from swh.deposit.constants import ( COL_IRI, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_REJECTED, diff --git a/swh/deposit/tests/api/test_deposit_private_list.py b/swh/deposit/tests/api/test_deposit_private_list.py --- a/swh/deposit/tests/api/test_deposit_private_list.py +++ b/swh/deposit/tests/api/test_deposit_private_list.py @@ -7,7 +7,7 @@ from rest_framework import status from swh.deposit.api.converters import convert_status_detail -from swh.deposit.config import ( +from swh.deposit.constants import ( DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_LOAD_SUCCESS, DEPOSIT_STATUS_PARTIAL, diff --git a/swh/deposit/tests/api/test_deposit_private_read_archive.py b/swh/deposit/tests/api/test_deposit_private_read_archive.py --- a/swh/deposit/tests/api/test_deposit_private_read_archive.py +++ b/swh/deposit/tests/api/test_deposit_private_read_archive.py @@ -10,7 +10,7 @@ from rest_framework import status from swh.deposit.api.private.deposit_read import aggregate_tarballs -from swh.deposit.config import EM_IRI, PRIVATE_GET_RAW_CONTENT +from swh.deposit.constants import EM_IRI, PRIVATE_GET_RAW_CONTENT from swh.deposit.tests.common import create_arborescence_archive PRIVATE_GET_RAW_CONTENT_NC = PRIVATE_GET_RAW_CONTENT + "-nc" diff --git a/swh/deposit/tests/api/test_deposit_private_read_metadata.py b/swh/deposit/tests/api/test_deposit_private_read_metadata.py --- a/swh/deposit/tests/api/test_deposit_private_read_metadata.py +++ b/swh/deposit/tests/api/test_deposit_private_read_metadata.py @@ -7,7 +7,8 @@ from rest_framework import status from swh.deposit import __version__ -from swh.deposit.config import PRIVATE_GET_DEPOSIT_METADATA, SE_IRI, SWH_PERSON +from swh.deposit.config import SWH_PERSON +from swh.deposit.constants import PRIVATE_GET_DEPOSIT_METADATA, SE_IRI from swh.deposit.models import Deposit from swh.deposit.parsers import parse_xml diff --git a/swh/deposit/tests/api/test_deposit_private_update_status.py b/swh/deposit/tests/api/test_deposit_private_update_status.py --- a/swh/deposit/tests/api/test_deposit_private_update_status.py +++ b/swh/deposit/tests/api/test_deposit_private_update_status.py @@ -10,7 +10,7 @@ from rest_framework import status from swh.deposit.api.private.deposit_update_status import MANDATORY_KEYS -from swh.deposit.config import ( +from swh.deposit.constants import ( DEPOSIT_STATUS_LOAD_FAILURE, DEPOSIT_STATUS_LOAD_SUCCESS, PRIVATE_PUT_DEPOSIT, diff --git a/swh/deposit/tests/api/test_deposit_schedule.py b/swh/deposit/tests/api/test_deposit_schedule.py --- a/swh/deposit/tests/api/test_deposit_schedule.py +++ b/swh/deposit/tests/api/test_deposit_schedule.py @@ -11,7 +11,7 @@ import pytest from rest_framework import status -from swh.deposit.config import ( +from swh.deposit.constants import ( COL_IRI, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_PARTIAL, diff --git a/swh/deposit/tests/api/test_deposit_state.py b/swh/deposit/tests/api/test_deposit_state.py --- a/swh/deposit/tests/api/test_deposit_state.py +++ b/swh/deposit/tests/api/test_deposit_state.py @@ -9,7 +9,7 @@ from django.urls import reverse_lazy as reverse from rest_framework import status -from swh.deposit.config import ( +from swh.deposit.constants import ( DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_REJECTED, STATE_IRI, diff --git a/swh/deposit/tests/api/test_deposit_update.py b/swh/deposit/tests/api/test_deposit_update.py --- a/swh/deposit/tests/api/test_deposit_update.py +++ b/swh/deposit/tests/api/test_deposit_update.py @@ -8,7 +8,7 @@ from django.urls import reverse_lazy as reverse from rest_framework import status -from swh.deposit.config import ( +from swh.deposit.constants import ( DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_PARTIAL, EDIT_IRI, diff --git a/swh/deposit/tests/api/test_deposit_update_atom.py b/swh/deposit/tests/api/test_deposit_update_atom.py --- a/swh/deposit/tests/api/test_deposit_update_atom.py +++ b/swh/deposit/tests/api/test_deposit_update_atom.py @@ -10,13 +10,13 @@ from rest_framework import status from swh.deposit.api.common import ACCEPT_ARCHIVE_CONTENT_TYPES -from swh.deposit.config import ( +from swh.deposit.config import APIConfig +from swh.deposit.constants import ( COL_IRI, DEPOSIT_STATUS_DEPOSITED, EDIT_IRI, EM_IRI, SE_IRI, - APIConfig, ) from swh.deposit.models import Deposit, DepositCollection, DepositRequest from swh.deposit.parsers import parse_xml diff --git a/swh/deposit/tests/api/test_deposit_update_binary.py b/swh/deposit/tests/api/test_deposit_update_binary.py --- a/swh/deposit/tests/api/test_deposit_update_binary.py +++ b/swh/deposit/tests/api/test_deposit_update_binary.py @@ -12,7 +12,7 @@ from rest_framework import status import xmltodict -from swh.deposit.config import COL_IRI, DEPOSIT_STATUS_DEPOSITED, EM_IRI, SE_IRI +from swh.deposit.constants import COL_IRI, DEPOSIT_STATUS_DEPOSITED, EM_IRI, SE_IRI from swh.deposit.models import Deposit, DepositRequest from swh.deposit.parsers import parse_xml from swh.deposit.tests.common import ( diff --git a/swh/deposit/tests/api/test_get_file.py b/swh/deposit/tests/api/test_get_file.py --- a/swh/deposit/tests/api/test_get_file.py +++ b/swh/deposit/tests/api/test_get_file.py @@ -8,7 +8,7 @@ from django.urls import reverse_lazy as reverse from rest_framework import status -from swh.deposit.config import CONT_FILE_IRI +from swh.deposit.constants import CONT_FILE_IRI from swh.deposit.models import DEPOSIT_STATUS_DETAIL from swh.deposit.parsers import parse_xml diff --git a/swh/deposit/tests/api/test_keycloak_auth.py b/swh/deposit/tests/api/test_keycloak_auth.py --- a/swh/deposit/tests/api/test_keycloak_auth.py +++ b/swh/deposit/tests/api/test_keycloak_auth.py @@ -9,7 +9,7 @@ import pytest from swh.auth.keycloak import KeycloakError -from swh.deposit.config import SD_IRI +from swh.deposit.constants import SD_IRI from swh.deposit.tests.conftest import mock_keycloakopenidconnect diff --git a/swh/deposit/tests/api/test_service_document.py b/swh/deposit/tests/api/test_service_document.py --- a/swh/deposit/tests/api/test_service_document.py +++ b/swh/deposit/tests/api/test_service_document.py @@ -6,7 +6,7 @@ from django.urls import reverse_lazy as reverse from rest_framework import status -from swh.deposit.config import SD_IRI +from swh.deposit.constants import SD_IRI def test_service_document_no_auth_fails(client): diff --git a/swh/deposit/tests/cli/test_admin.py b/swh/deposit/tests/cli-admin/test_admin.py rename from swh/deposit/tests/cli/test_admin.py rename to swh/deposit/tests/cli-admin/test_admin.py --- a/swh/deposit/tests/cli/test_admin.py +++ b/swh/deposit/tests/cli-admin/test_admin.py @@ -3,10 +3,11 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from click.testing import CliRunner import pytest from swh.deposit.cli.admin import admin as cli -from swh.deposit.config import ( +from swh.deposit.constants import ( DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_PARTIAL, DEPOSIT_STATUS_VERIFIED, @@ -20,54 +21,54 @@ pass -def test_cli_admin_user_list_nothing(cli_runner): - result = cli_runner.invoke(cli, ["user", "list",]) +def test_cli_admin_user_list_nothing(): + result = CliRunner().invoke(cli, ["user", "list",]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert result.output == "Empty user list\n" -def test_cli_admin_user_list_with_users(cli_runner, deposit_user): - result = cli_runner.invoke(cli, ["user", "list",]) +def test_cli_admin_user_list_with_users(deposit_user): + result = CliRunner().invoke(cli, ["user", "list",]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert result.output == f"{deposit_user.username}\n" # only 1 user -def test_cli_admin_collection_list_nothing(cli_runner): - result = cli_runner.invoke(cli, ["collection", "list",]) +def test_cli_admin_collection_list_nothing(): + result = CliRunner().invoke(cli, ["collection", "list",]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert result.output == "Empty collection list\n" -def test_cli_admin_collection_list_with_collections(cli_runner, deposit_collection): +def test_cli_admin_collection_list_with_collections(deposit_collection): from swh.deposit.tests.conftest import create_deposit_collection new_collection = create_deposit_collection("something") - result = cli_runner.invoke(cli, ["collection", "list",]) + result = CliRunner().invoke(cli, ["collection", "list",]) assert result.exit_code == 0, f"Unexpected output: {result.output}" collections = "\n".join([deposit_collection.name, new_collection.name]) assert result.output == f"{collections}\n" -def test_cli_admin_user_exists_unknown(cli_runner): - result = cli_runner.invoke(cli, ["user", "exists", "unknown"]) +def test_cli_admin_user_exists_unknown(): + result = CliRunner().invoke(cli, ["user", "exists", "unknown"]) assert result.exit_code == 1, f"Unexpected output: {result.output}" assert result.output == "User unknown does not exist.\n" -def test_cli_admin_user_exists(cli_runner, deposit_user): - result = cli_runner.invoke(cli, ["user", "exists", deposit_user.username]) +def test_cli_admin_user_exists(deposit_user): + result = CliRunner().invoke(cli, ["user", "exists", deposit_user.username]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert result.output == f"User {deposit_user.username} exists.\n" -def test_cli_admin_create_collection(cli_runner): +def test_cli_admin_create_collection(): collection_name = "something" try: @@ -75,7 +76,7 @@ except DepositCollection.DoesNotExist: pass - result = cli_runner.invoke( + result = CliRunner().invoke( cli, ["collection", "create", "--name", collection_name,] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" @@ -90,7 +91,7 @@ """ ) - result2 = cli_runner.invoke( + result2 = CliRunner().invoke( cli, ["collection", "create", "--name", collection_name,] ) assert result2.exit_code == 0, f"Unexpected output: {result.output}" @@ -101,7 +102,7 @@ ) -def test_cli_admin_user_create(cli_runner): +def test_cli_admin_user_create(): user_name = "user" collection_name = user_name @@ -115,7 +116,7 @@ except DepositCollection.DoesNotExist: pass - result = cli_runner.invoke( + result = CliRunner().invoke( cli, ["user", "create", "--username", user_name, "--password", "password",] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" @@ -146,7 +147,7 @@ assert user.last_name == "" # create a user that already exists - result2 = cli_runner.invoke( + result2 = CliRunner().invoke( cli, [ "user", @@ -197,7 +198,7 @@ ) -def test_cli_admin_reschedule_unknown_deposit(cli_runner): +def test_cli_admin_reschedule_unknown_deposit(): """Rescheduling unknown deposit should report failure """ @@ -210,7 +211,7 @@ except Deposit.DoesNotExist: pass - result = cli_runner.invoke( + result = CliRunner().invoke( cli, ["deposit", "reschedule", "--deposit-id", unknown_deposit_id] ) @@ -218,7 +219,7 @@ assert result.exit_code == 1 -def test_cli_admin_reschedule_verified_deposit(cli_runner, complete_deposit): +def test_cli_admin_reschedule_verified_deposit(complete_deposit): """Rescheduling verified deposit should do nothing but report """ @@ -226,7 +227,7 @@ deposit.status = "verified" deposit.save() - result = cli_runner.invoke( + result = CliRunner().invoke( cli, ["deposit", "reschedule", "--deposit-id", deposit.id] ) @@ -238,7 +239,7 @@ "status_to_check", [DEPOSIT_STATUS_PARTIAL, DEPOSIT_STATUS_DEPOSITED] ) def test_cli_admin_reschedule_unaccepted_deposit_status( - status_to_check, cli_runner, complete_deposit + status_to_check, complete_deposit ): """Rescheduling verified deposit should do nothing but report @@ -247,7 +248,7 @@ deposit.status = status_to_check # not accepted status will fail the check deposit.save() - result = cli_runner.invoke( + result = CliRunner().invoke( cli, ["deposit", "reschedule", "--deposit-id", deposit.id] ) @@ -258,7 +259,7 @@ assert result.exit_code == 1 -def test_cli_admin_reschedule_missing_task_id(cli_runner, complete_deposit): +def test_cli_admin_reschedule_missing_task_id(complete_deposit): """Rescheduling deposit with no load_task_id cannot work. """ @@ -266,7 +267,7 @@ deposit.load_task_id = "" # drop the load-task-id so it fails the check deposit.save() - result = cli_runner.invoke( + result = CliRunner().invoke( cli, ["deposit", "reschedule", "--deposit-id", deposit.id] ) @@ -277,7 +278,7 @@ assert result.exit_code == 1 -def test_cli_admin_reschedule_nominal(cli_runner, complete_deposit, swh_scheduler): +def test_cli_admin_reschedule_nominal(complete_deposit, swh_scheduler): """Rescheduling deposit with no load_task_id cannot work. """ @@ -301,7 +302,7 @@ deposit.save() # Reschedule it - result = cli_runner.invoke( + result = CliRunner().invoke( cli, ["deposit", "reschedule", "--deposit-id", deposit.id] ) assert result.exit_code == 0 diff --git a/swh/deposit/tests/cli/conftest.py b/swh/deposit/tests/cli/conftest.py deleted file mode 100644 --- a/swh/deposit/tests/cli/conftest.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (C) 2019-2020 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from click.testing import CliRunner -import pytest - - -@pytest.fixture -def cli_runner(): - return CliRunner() diff --git a/swh/deposit/tests/cli/test_client.py b/swh/deposit/tests/cli/test_client.py --- a/swh/deposit/tests/cli/test_client.py +++ b/swh/deposit/tests/cli/test_client.py @@ -12,6 +12,7 @@ from unittest.mock import MagicMock from xml.etree import ElementTree +from click.testing import CliRunner import pytest import yaml @@ -28,11 +29,13 @@ PublicApiDepositClient, ServiceDocumentDepositClient, ) -from swh.deposit.parsers import parse_xml +from swh.deposit.utils import parse_xml from swh.model.exceptions import ValidationError from ..conftest import TEST_USER +pytestmark = pytest.mark.client + def generate_slug() -> str: """Generate a slug (sample purposes). @@ -107,7 +110,7 @@ def test_cli_upload_conflictual_flags( - datadir, requests_mock_datadir, cli_runner, atom_dataset, tmp_path, + datadir, requests_mock_datadir, atom_dataset, tmp_path, ): """Post metadata-only deposit through cli with invalid swhid raises @@ -120,7 +123,7 @@ with pytest.raises(InputError, match="both with different values"): # fmt: off - cli_runner.invoke( + CliRunner().invoke( cli, [ "upload", @@ -138,13 +141,13 @@ def test_cli_deposit_with_server_down_for_maintenance( - sample_archive, caplog, client_mock_api_down, slug, patched_tmp_path, cli_runner + sample_archive, caplog, client_mock_api_down, slug, patched_tmp_path ): """ Deposit failure due to maintenance down time should be explicit """ # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -239,7 +242,7 @@ def test_cli_single_minimal_deposit_with_slug( - sample_archive, slug, patched_tmp_path, requests_mock_datadir, cli_runner, caplog, + sample_archive, slug, patched_tmp_path, requests_mock_datadir, caplog, ): """ This ensure a single deposit upload through the cli is fine, cf. https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit @@ -247,7 +250,7 @@ metadata_path = os.path.join(patched_tmp_path, "metadata.xml") # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -293,7 +296,7 @@ def test_cli_single_minimal_deposit_with_create_origin( - sample_archive, slug, patched_tmp_path, requests_mock_datadir, cli_runner, caplog, + sample_archive, slug, patched_tmp_path, requests_mock_datadir, caplog, ): """ This ensure a single deposit upload through the cli is fine, cf. https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit @@ -303,7 +306,7 @@ origin = slug # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -355,9 +358,7 @@ ), "We should have no warning as we are using create_origin" -def test_cli_validation_metadata( - sample_archive, caplog, patched_tmp_path, cli_runner, slug -): +def test_cli_validation_metadata(sample_archive, caplog, patched_tmp_path, slug): """Multiple metadata flags scenario (missing, conflicts) properly fails the calls """ @@ -372,7 +373,7 @@ ]: # Test missing author then missing name # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -406,7 +407,7 @@ # incompatible flags: Test both --metadata and --author, then --metadata and # --name # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -439,7 +440,7 @@ # incompatible flags check (Test both --metadata and --author, # then --metadata and --name) # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -470,13 +471,13 @@ caplog.clear() -def test_cli_validation_no_actionable_command(caplog, cli_runner): +def test_cli_validation_no_actionable_command(caplog): """Multiple metadata flags scenario (missing, conflicts) properly fails the calls """ # no actionable command # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -502,7 +503,7 @@ def test_cli_validation_replace_with_no_deposit_id_fails( - sample_archive, caplog, patched_tmp_path, requests_mock_datadir, datadir, cli_runner + sample_archive, caplog, patched_tmp_path, requests_mock_datadir, datadir ): """--replace flags require --deposit-id otherwise fails @@ -510,7 +511,7 @@ metadata_path = os.path.join(datadir, "atom", "entry-data-deposit-binary.xml") # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -538,14 +539,14 @@ def test_cli_single_deposit_slug_generation( - sample_archive, patched_tmp_path, requests_mock_datadir, cli_runner + sample_archive, patched_tmp_path, requests_mock_datadir ): """Single deposit scenario without providing the slug, it should not be generated. """ metadata_path = os.path.join(patched_tmp_path, "metadata.xml") # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -573,9 +574,7 @@ assert "codemeta:identifier" not in actual_metadata -def test_cli_multisteps_deposit( - sample_archive, datadir, slug, requests_mock_datadir, cli_runner -): +def test_cli_multisteps_deposit(sample_archive, datadir, slug, requests_mock_datadir): """ First deposit a partial deposit (no metadata, only archive), then update the metadata part. https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#multisteps-deposit """ # noqa @@ -584,7 +583,7 @@ # Create a partial deposit with only 1 archive # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -610,7 +609,7 @@ # Update the partial deposit with only 1 archive # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -638,7 +637,7 @@ # Update deposit with metadata # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -676,7 +675,7 @@ ], ) def test_cli_deposit_status_with_output_format( - output_format, parser_fn, datadir, slug, requests_mock_datadir, caplog, cli_runner + output_format, parser_fn, datadir, slug, requests_mock_datadir, caplog ): """Check deposit status cli with all possible output formats (json, yaml, logging). @@ -696,7 +695,7 @@ } # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "status", @@ -722,7 +721,7 @@ def test_cli_update_metadata_with_swhid_on_completed_deposit( - datadir, requests_mock_datadir, cli_runner + datadir, requests_mock_datadir ): """Update new metadata on a completed deposit (status done) is ok """ @@ -744,7 +743,7 @@ assert expected_deposit_status["deposit_swh_id"] is not None # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -766,7 +765,7 @@ def test_cli_update_metadata_with_swhid_on_other_status_deposit( - datadir, requests_mock_datadir, cli_runner + datadir, requests_mock_datadir ): """Update new metadata with swhid on other deposit status is not possible """ @@ -774,7 +773,7 @@ deposit_id = "321" # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "upload", @@ -801,7 +800,7 @@ def test_cli_metadata_only_deposit_full_metadata_file( - datadir, requests_mock_datadir, cli_runner, atom_dataset, tmp_path, + datadir, requests_mock_datadir, atom_dataset, tmp_path, ): """Post metadata-only deposit through cli @@ -824,7 +823,7 @@ assert expected_deposit_status["deposit_status"] == "done" # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "metadata-only", @@ -843,7 +842,7 @@ def test_cli_metadata_only_deposit_invalid_swhid( - datadir, requests_mock_datadir, cli_runner, atom_dataset, tmp_path, + datadir, requests_mock_datadir, atom_dataset, tmp_path, ): """Post metadata-only deposit through cli with invalid swhid raises @@ -857,7 +856,7 @@ with pytest.raises(ValidationError, match="Invalid"): # fmt: off - cli_runner.invoke( + CliRunner().invoke( cli, [ "metadata-only", @@ -873,7 +872,7 @@ def test_cli_metadata_only_deposit_no_swhid( - datadir, requests_mock_datadir, cli_runner, atom_dataset, tmp_path, + datadir, requests_mock_datadir, atom_dataset, tmp_path, ): """Post metadata-only deposit through cli with invalid swhid raises @@ -886,7 +885,7 @@ with pytest.raises(InputError, match="SWHID must be provided"): # fmt: off - cli_runner.invoke( + CliRunner().invoke( cli, [ "metadata-only", @@ -905,12 +904,7 @@ "metadata_entry_key", ["entry-data-with-add-to-origin", "entry-only-create-origin"] ) def test_cli_deposit_warning_missing_origin( - metadata_entry_key, - tmp_path, - atom_dataset, - caplog, - cli_runner, - requests_mock_datadir, + metadata_entry_key, tmp_path, atom_dataset, caplog, requests_mock_datadir, ): """Deposit cli should warn when provided metadata xml is missing 'origins' tags @@ -925,7 +919,7 @@ f.write(metadata_raw) # fmt: off - cli_runner.invoke( + CliRunner().invoke( cli, [ "upload", @@ -943,7 +937,7 @@ def test_cli_deposit_warning_missing_provenance_url( - tmp_path, atom_dataset, caplog, cli_runner, requests_mock_datadir, + tmp_path, atom_dataset, caplog, requests_mock_datadir, ): """Deposit cli should warn when no metadata provenance is provided @@ -955,7 +949,7 @@ f.write(metadata_raw) # fmt: off - cli_runner.invoke( + CliRunner().invoke( cli, [ "upload", @@ -1021,7 +1015,7 @@ ], ) def test_cli_deposit_collection_list( - output_format, parser_fn, datadir, slug, requests_mock_datadir, caplog, cli_runner + output_format, parser_fn, datadir, slug, requests_mock_datadir, caplog ): """Check deposit status cli with all possible output formats (json, yaml, logging). @@ -1058,7 +1052,7 @@ } # fmt: off - result = cli_runner.invoke( + result = CliRunner().invoke( cli, [ "list", diff --git a/swh/deposit/tests/client/__init__.py b/swh/deposit/tests/client/__init__.py new file mode 100644 diff --git a/swh/deposit/tests/client/data b/swh/deposit/tests/client/data new file mode 120000 --- /dev/null +++ b/swh/deposit/tests/client/data @@ -0,0 +1 @@ +../data \ No newline at end of file diff --git a/swh/deposit/tests/test_client_module.py b/swh/deposit/tests/client/test_client_module.py rename from swh/deposit/tests/test_client_module.py rename to swh/deposit/tests/client/test_client_module.py diff --git a/swh/deposit/tests/common.py b/swh/deposit/tests/common.py --- a/swh/deposit/tests/common.py +++ b/swh/deposit/tests/common.py @@ -10,8 +10,6 @@ import tarfile import tempfile -from django.core.files.uploadedfile import InMemoryUploadedFile - from swh.core import tarball @@ -181,6 +179,8 @@ def _post_or_put_multipart(f, url, archive, atom_entry, **kwargs): + from django.core.files.uploadedfile import InMemoryUploadedFile + archive = InMemoryUploadedFile( BytesIO(archive["data"]), field_name=archive["name"], diff --git a/swh/deposit/tests/conftest.py b/swh/deposit/tests/conftest.py --- a/swh/deposit/tests/conftest.py +++ b/swh/deposit/tests/conftest.py @@ -6,25 +6,35 @@ import base64 from copy import deepcopy from functools import partial -from io import BytesIO import os import re from typing import TYPE_CHECKING, Dict, Mapping +from xml.etree import ElementTree + +try: + import django +except ImportError: + django = None # type: ignore # probably running clientonly tests +else: + from django.test.utils import setup_databases # type: ignore + from django.urls import reverse_lazy as reverse + from rest_framework import status + from rest_framework.test import APIClient -from django.test.utils import setup_databases # type: ignore -from django.urls import reverse_lazy as reverse -import psycopg2 -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT import pytest -from rest_framework import status -from rest_framework.test import APIClient import yaml -from swh.auth.pytest_plugin import keycloak_mock_factory +if django: + from swh.auth.pytest_plugin import keycloak_mock_factory + from swh.core.config import read from swh.core.pytest_plugin import get_response_cb -from swh.deposit.auth import DEPOSIT_PERMISSION -from swh.deposit.config import ( + +if django: + from swh.deposit.auth import DEPOSIT_PERMISSION + from swh.deposit.config import setup_django_for + +from swh.deposit.constants import ( COL_IRI, DEPOSIT_STATUS_DEPOSITED, DEPOSIT_STATUS_LOAD_FAILURE, @@ -33,21 +43,35 @@ DEPOSIT_STATUS_REJECTED, DEPOSIT_STATUS_VERIFIED, SE_IRI, - setup_django_for, -) -from swh.deposit.parsers import parse_xml -from swh.deposit.tests.common import ( - create_arborescence_archive, - post_archive, - post_atom, ) +from swh.deposit.utils import NAMESPACES from swh.model.hashutil import hash_to_bytes from swh.model.swhids import CoreSWHID, ObjectType, QualifiedSWHID -from swh.scheduler import get_scheduler if TYPE_CHECKING: from swh.deposit.models import Deposit, DepositClient, DepositCollection +pytest_plugins = [ + "swh.core.pytest_plugin", +] + +try: + import swh.scheduler + import swh.storage # noqa +except ImportError: + # probably running client-only tests + pass +else: + pytest_plugins.append("swh.scheduler.pytest_plugin") + pytest_plugins.append("swh.storage.pytest_plugin") + + +@pytest.fixture(scope="session") +def swh_scheduler_celery_includes(swh_scheduler_celery_includes): + return swh_scheduler_celery_includes + [ + "swh.deposit.loader.tasks", + ] + # mypy is asked to ignore the import statement above because setup_databases # is not part of the d.t.utils.__all__ variable. @@ -94,26 +118,27 @@ CLIENT_ID = "swh-deposit" -keycloak_mock_auth_success = keycloak_mock_factory( - server_url=KEYCLOAK_SERVER_URL, - realm_name=KEYCLOAK_REALM_NAME, - client_id=CLIENT_ID, - auth_success=True, - user_info=USER_INFO, - client_permissions=[DEPOSIT_PERMISSION], -) - +if django: + keycloak_mock_auth_success = keycloak_mock_factory( + server_url=KEYCLOAK_SERVER_URL, + realm_name=KEYCLOAK_REALM_NAME, + client_id=CLIENT_ID, + auth_success=True, + user_info=USER_INFO, + client_permissions=[DEPOSIT_PERMISSION], + ) -keycloak_mock_auth_failure = keycloak_mock_factory( - server_url=KEYCLOAK_SERVER_URL, - realm_name=KEYCLOAK_REALM_NAME, - client_id=CLIENT_ID, - auth_success=False, -) + keycloak_mock_auth_failure = keycloak_mock_factory( + server_url=KEYCLOAK_SERVER_URL, + realm_name=KEYCLOAK_REALM_NAME, + client_id=CLIENT_ID, + auth_success=False, + ) def pytest_configure(): - setup_django_for("testing") + if django: + setup_django_for("testing") @pytest.fixture @@ -127,91 +152,83 @@ return requests_mock_datadir -@pytest.fixture -def common_deposit_config(swh_scheduler_config, swh_storage_backend_config): - return { - "max_upload_size": 500, - "extraction_dir": "/tmp/swh-deposit/test/extraction-dir", - "checks": False, - "scheduler": {"cls": "local", **swh_scheduler_config,}, - "storage": swh_storage_backend_config, - "storage_metadata": swh_storage_backend_config, - "swh_authority_url": "http://deposit.softwareheritage.example/", - } - - -@pytest.fixture() -def deposit_config(common_deposit_config): - return { - **common_deposit_config, - "authentication_provider": "keycloak", - "keycloak": { - "server_url": KEYCLOAK_SERVER_URL, - "realm_name": KEYCLOAK_REALM_NAME, - }, - } +if django: + @pytest.fixture + def common_deposit_config(swh_scheduler_config, swh_storage_backend_config): + return { + "max_upload_size": 500, + "extraction_dir": "/tmp/swh-deposit/test/extraction-dir", + "checks": False, + "scheduler": {"cls": "local", **swh_scheduler_config,}, + "storage": swh_storage_backend_config, + "storage_metadata": swh_storage_backend_config, + "swh_authority_url": "http://deposit.softwareheritage.example/", + } -@pytest.fixture() -def deposit_config_path(tmp_path, monkeypatch, deposit_config): - conf_path = os.path.join(tmp_path, "deposit.yml") - with open(conf_path, "w") as f: - f.write(yaml.dump(deposit_config)) - monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path) - return conf_path - - -@pytest.fixture(autouse=True) -def deposit_autoconfig(deposit_config_path): - """Enforce config for deposit classes inherited from APIConfig.""" - cfg = read(deposit_config_path) - - if "scheduler" in cfg: - # scheduler setup: require the check-deposit and load-deposit tasks - scheduler = get_scheduler(**cfg["scheduler"]) - task_types = [ - { - "type": "check-deposit", - "backend_name": "swh.deposit.loader.tasks.ChecksDepositTsk", - "description": "Check deposit metadata/archive before loading", - "num_retries": 3, - }, - { - "type": "load-deposit", - "backend_name": "swh.loader.package.deposit.tasks.LoadDeposit", - "description": "Loading deposit archive into swh archive", - "num_retries": 3, + @pytest.fixture() + def deposit_config(common_deposit_config): + return { + **common_deposit_config, + "authentication_provider": "keycloak", + "keycloak": { + "server_url": KEYCLOAK_SERVER_URL, + "realm_name": KEYCLOAK_REALM_NAME, }, - ] - for task_type in task_types: - scheduler.create_task_type(task_type) - - -@pytest.fixture(scope="session") -def django_db_setup(request, django_db_blocker, postgresql_proc): - from django.conf import settings - - settings.DATABASES["default"].update( - { - ("ENGINE", "django.db.backends.postgresql"), - ("NAME", "tests"), - ("USER", postgresql_proc.user), # noqa - ("HOST", postgresql_proc.host), # noqa - ("PORT", postgresql_proc.port), # noqa } - ) - with django_db_blocker.unblock(): - setup_databases( - verbosity=request.config.option.verbose, interactive=False, keepdb=False - ) - -def execute_sql(sql): - """Execute sql to postgres db""" - with psycopg2.connect(database="postgres") as conn: - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - cur = conn.cursor() - cur.execute(sql) + @pytest.fixture() + def deposit_config_path(tmp_path, monkeypatch, deposit_config): + conf_path = os.path.join(tmp_path, "deposit.yml") + with open(conf_path, "w") as f: + f.write(yaml.dump(deposit_config)) + monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path) + return conf_path + + @pytest.fixture(autouse=True) + def deposit_autoconfig(deposit_config_path): + """Enforce config for deposit classes inherited from APIConfig.""" + cfg = read(deposit_config_path) + + if "scheduler" in cfg: + from swh.scheduler import get_scheduler + + # scheduler setup: require the check-deposit and load-deposit tasks + scheduler = get_scheduler(**cfg["scheduler"]) + task_types = [ + { + "type": "check-deposit", + "backend_name": "swh.deposit.loader.tasks.ChecksDepositTsk", + "description": "Check deposit metadata/archive before loading", + "num_retries": 3, + }, + { + "type": "load-deposit", + "backend_name": "swh.loader.package.deposit.tasks.LoadDeposit", + "description": "Loading deposit archive into swh archive", + "num_retries": 3, + }, + ] + for task_type in task_types: + scheduler.create_task_type(task_type) + + @pytest.fixture(scope="session") + def django_db_setup(request, django_db_blocker, postgresql_proc): + from django.conf import settings + + settings.DATABASES["default"].update( + { + ("ENGINE", "django.db.backends.postgresql"), + ("NAME", "tests"), + ("USER", postgresql_proc.user), # noqa + ("HOST", postgresql_proc.host), # noqa + ("PORT", postgresql_proc.port), # noqa + } + ) + with django_db_blocker.unblock(): + setup_databases( + verbosity=request.config.option.verbose, interactive=False, keepdb=False + ) @pytest.fixture(autouse=True, scope="session") @@ -373,6 +390,8 @@ """Returns a sample archive """ + from swh.deposit.tests.common import create_arborescence_archive + tmp_path = str(tmp_path) # pytest version limitation in previous version archive = create_arborescence_archive( tmp_path, "archive1", "file1", b"some content in file" @@ -432,6 +451,9 @@ """Create a skeleton shell deposit """ + from swh.deposit.models import Deposit + from swh.deposit.tests.common import post_archive + url = reverse(COL_IRI, args=[collection_name]) # when response = post_archive( @@ -444,10 +466,11 @@ # then assert response.status_code == status.HTTP_201_CREATED, response.content.decode() - from swh.deposit.models import Deposit - response_content = parse_xml(BytesIO(response.content)) - deposit_id = response_content["swh:deposit_id"] + response_content = ElementTree.fromstring(response.content) + elt = response_content.find("swh:deposit_id", namespaces=NAMESPACES) + assert elt is not None and elt.text is not None + deposit_id = int(elt.text) deposit = Deposit._default_manager.get(id=deposit_id) if deposit.status != deposit_status: @@ -468,6 +491,8 @@ to `deposit_status`. """ + from swh.deposit.tests.common import post_atom + deposit = create_deposit( authenticated_client, collection_name, @@ -552,6 +577,7 @@ def partial_deposit_only_metadata( deposit_collection, authenticated_client, atom_dataset ): + from swh.deposit.tests.common import post_atom response = post_atom( authenticated_client, @@ -563,8 +589,8 @@ assert response.status_code == status.HTTP_201_CREATED - response_content = parse_xml(response.content) - deposit_id = response_content["swh:deposit_id"] + response_content = ElementTree.fromstring(response.content) + deposit_id = response_content.find("swh:deposit_id", namespaces=NAMESPACES).text from swh.deposit.models import Deposit deposit = Deposit._default_manager.get(pk=deposit_id) diff --git a/swh/deposit/tests/loader/test_client.py b/swh/deposit/tests/loader/test_client.py --- a/swh/deposit/tests/loader/test_client.py +++ b/swh/deposit/tests/loader/test_client.py @@ -12,7 +12,10 @@ from requests import Session from swh.deposit.client import PrivateApiDepositClient -from swh.deposit.config import DEPOSIT_STATUS_LOAD_FAILURE, DEPOSIT_STATUS_LOAD_SUCCESS +from swh.deposit.constants import ( + DEPOSIT_STATUS_LOAD_FAILURE, + DEPOSIT_STATUS_LOAD_SUCCESS, +) CLIENT_TEST_CONFIG = { "url": "https://nowhere.org/", diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -1,23 +1,25 @@ [tox] -envlist=flake8,mypy,py3-django2,py3-django3 +envlist=flake8,mypy,py3-django2,py3-django3,py3-clientonly [testenv] extras = - testing + testing-core + !clientonly: testing deps = # the dependency below is needed for now as a workaround for # https://github.com/pypa/pip/issues/6239 swh.core[http] >= 0.3 - swh.scheduler[testing] >= 0.5.0 + !clientonly: swh.scheduler[testing] >= 0.5.0 dev: pdbpp pytest-cov django2: Django>=2,<3 django3: Django>=3,<4 commands = pytest \ - !dev: --cov {envsitepackagesdir}/swh/deposit --cov-branch \ - {envsitepackagesdir}/swh/deposit \ - {posargs} + clientonly: {envsitepackagesdir}/swh/deposit/tests/cli/ {envsitepackagesdir}/swh/deposit/tests/client/ \ + !clientonly: {envsitepackagesdir}/swh/deposit/ \ + !clientonly-!dev: --cov {envsitepackagesdir}/swh/deposit --cov-branch \ + {posargs} [testenv:black] skip_install = true