(swh) boris@debian:~/swh-environment/swh-loader-core/swh/loader/package/jar$ pytest -v . ================================================ test session starts ================================================= platform linux -- Python 3.7.3, pytest-6.2.4, py-1.10.0, pluggy-0.13.1 -- /home/boris/.virtualenvs/swh/bin/python3 cachedir: .pytest_cache hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/home/boris/swh-environment/swh-loader-core/swh/loader/package/jar/.hypothesis/examples') rootdir: /home/boris/swh-environment/swh-loader-core, configfile: pytest.ini plugins: django-test-migrations-1.1.0, dash-1.21.0, xdist-2.3.0, redis-2.1.1, requests-mock-1.9.3, hypothesis-6.15.0, mock-3.6.1, asyncio-0.15.1, flask-1.2.0, django-4.4.0, forked-1.3.0, postgresql-2.6.1, swh.core-0.14.4, swh.journal-0.8.0 collected 5 items tests/test_jar.py::test_jar_visit_with_no_artifact_found PASSED [ 20%] tests/test_jar.py::test_jar_visit_with_release_artifact_no_prior_visit FAILED [ 40%] tests/test_jar.py::test_jar_2_visits_without_change FAILED [ 60%] tests/test_tasks.py::test_tasks_jar_loader FAILED [ 80%] tests/test_tasks.py::test_tasks_jar_loader_snapshot_append FAILED [100%] ====================================================== FAILURES ====================================================== ________________________________ test_jar_visit_with_release_artifact_no_prior_visit _________________________________ swh_storage = requests_mock = data_jar_1 = b'PK\x03\x04\n\x00\x00\x08\x08\x00\xba\\\xb7L\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\t\x00\x00\x00META-INF/\x...*\x00\x00al/aldi/sprova4j/models/Execution.javaPK\x05\x06\x00\x00\x00\x00#\x00#\x00\xe2\n\x00\x00\xf4,\x00\x00\x00\x00' def test_jar_visit_with_release_artifact_no_prior_visit( swh_storage, requests_mock, data_jar_1 ): """With no prior visit, loading a jar ends up with 1 snapshot """ requests_mock.get(MVN_ARTIFACTS[0]["url"], content=data_jar_1) loader = JarLoader( swh_storage, MVN_ARTIFACTS[0]["url"], artifacts=[MVN_ARTIFACTS[0]] ) actual_load_status = loader.load() print(f"LOAD STATUS {actual_load_status.items()}") print(f"LOAD STATUS {actual_load_status.keys()}") # FAIL should be eventful assert actual_load_status["status"] == "uneventful" expected_snapshot_first_visit_id = hash_to_bytes( "1a8893e6a86f444e8be8e7bda6cb34fb1735a00e" ) assert ( hash_to_bytes(actual_load_status["snapshot_id"]) == expected_snapshot_first_visit_id ) print(f"Debug actual load status {actual_load_status}") stats = get_stats(swh_storage) print(f"Debug stats {stats}") # FAIL status should be full assert_last_visit_matches( swh_storage, MVN_ARTIFACTS[0]["url"], status="partial", type="jar" ) > assert { "content": len(_expected_new_contents_first_visit), "directory": len(_expected_new_directories_first_visit), "origin": 1, "origin_visit": 1, "release": 0, "revision": len(_expected_new_revisions_first_visit), "skipped_content": 0, "snapshot": 1, } == stats E AssertionError: assert {'content': 2...isit': 1, ...} == {'content': 2...isit': 1, ...} E Omitting 7 identical items, use -vv to show E Differing items: E {'revision': 1} != {'revision': 0} E Full diff: E { E 'content': 26, E 'directory': 9,... E E ...Full output truncated (11 lines hidden), use '-vv' to show tests/test_jar.py:175: AssertionError ------------------------------------------------ Captured stdout call ------------------------------------------------ In get_versions [{'time': 1626109619335, 'url': 'https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/sprova4j-0.1.0.jar', 'gid': 'al.aldi', 'aid': 'sprova4j', 'filename': 'sprova4j-0.1.0.jar', 'version': '0.1.0'}] In get_versions versions ['0.1.0'] In get_package_info 0.1.0 Archive: /tmp/tmpubiyikmh/sprova4j-0.1.0.jar creating: /tmp/tmpubiyikmh/src/META-INF/ inflating: /tmp/tmpubiyikmh/src/META-INF/MANIFEST.MF inflating: /tmp/tmpubiyikmh/src/logback.xml inflating: /tmp/tmpubiyikmh/src/logback-test.xml creating: /tmp/tmpubiyikmh/src/al/ creating: /tmp/tmpubiyikmh/src/al/aldi/ creating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/ inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/Connector.java creating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/utils/ inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/utils/LoggingInterceptor.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/utils/AuthorizationInterceptor.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/utils/Utils.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/utils/SprovaObjectFilter.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/utils/ApiUtils.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/TestManager.java creating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/exections/ inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/exections/TestCaseException.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/exections/CycleException.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/exections/SprovaClientException.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/exections/ExecutionException.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/Main.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/SprovaLogger.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/SprovaApiClient.java creating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/ inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/TestStep.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/Cycle.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/TestCase.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/Project.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/SprovaObject.java creating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/aux/ inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/aux/PutExecutionResponse.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/aux/AuthenticationResponse.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/aux/AuthenticationRequest.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/User.java inflating: /tmp/tmpubiyikmh/src/al/aldi/sprova4j/models/Execution.java In build_revision JarPackageInfo(url='https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/sprova4j-0.1.0.jar', filename='sprova4j-0.1.0.jar', directory_extrinsic_metadata=[], time=1626109619335, raw_info={'time': 1626109619335, 'url': 'https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/sprova4j-0.1.0.jar', 'gid': 'al.aldi', 'aid': 'sprova4j', 'filename': 'sprova4j-0.1.0.jar', 'version': '0.1.0'}, gid='al.aldi', aid='sprova4j', version='0.1.0'). In get_default_versions LOAD STATUS dict_items([('status', 'uneventful'), ('snapshot_id', '1a8893e6a86f444e8be8e7bda6cb34fb1735a00e')]) LOAD STATUS dict_keys(['status', 'snapshot_id']) Debug actual load status {'status': 'uneventful', 'snapshot_id': '1a8893e6a86f444e8be8e7bda6cb34fb1735a00e'} Debug stats {'content': 26, 'directory': 9, 'origin': 1, 'origin_visit': 1, 'release': 0, 'revision': 0, 'skipped_content': 0, 'snapshot': 1} ------------------------------------------------- Captured log call -------------------------------------------------- WARNING swh.storage.proxies.retry:retry.py:48 Retrying RPC call WARNING swh.storage.proxies.retry:retry.py:48 Retrying RPC call WARNING swh.storage.proxies.retry:retry.py:48 Retrying RPC call ERROR swh.loader.package.loader:loader.py:610 Failed to load branch 0.1.0 for https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/sprova4j-0.1.0.jar Traceback (most recent call last): File "/home/boris/.virtualenvs/swh/lib/python3.7/site-packages/tenacity/__init__.py", line 407, in __call__ result = fn(*args, **kwargs) File "/home/boris/swh-environment/swh-storage/swh/storage/proxies/retry.py", line 67, in newf return getattr(storage, attribute_name)(*args, **kwargs) File "/home/boris/swh-environment/swh-storage/swh/storage/proxies/buffer.py", line 156, in flush stats = add_fn(list(batch)) File "/home/boris/swh-environment/swh-storage/swh/storage/metrics.py", line 24, in d return f(*a, **kw) File "/home/boris/swh-environment/swh-storage/swh/storage/metrics.py", line 77, in d r = f(*a, **kw) File "/home/boris/swh-environment/swh-core/swh/core/db/common.py", line 62, in _meth return meth(self, *args, db=db, cur=cur, **kwargs) File "/home/boris/swh-environment/swh-storage/swh/storage/postgresql/storage.py", line 621, in revision_add db_revisions_filtered = list(map(converters.revision_to_db, revisions_filtered)) File "/home/boris/swh-environment/swh-storage/swh/storage/postgresql/converters.py", line 153, in revision_to_db date = date_to_db(revision.date) File "/home/boris/swh-environment/swh-storage/swh/storage/postgresql/converters.py", line 137, in date_to_db timestamp = datetime.datetime.fromtimestamp(ts.seconds, datetime.timezone.utc) ValueError: year 53499 is out of range The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/home/boris/swh-environment/swh-loader-core/swh/loader/package/loader.py", line 603, in load self.storage.flush() File "/home/boris/.virtualenvs/swh/lib/python3.7/site-packages/tenacity/__init__.py", line 324, in wrapped_f return self(f, *args, **kw) File "/home/boris/.virtualenvs/swh/lib/python3.7/site-packages/tenacity/__init__.py", line 404, in __call__ do = self.iter(retry_state=retry_state) File "/home/boris/.virtualenvs/swh/lib/python3.7/site-packages/tenacity/__init__.py", line 361, in iter raise retry_exc from fut.exception() tenacity.RetryError: RetryError[] WARNING swh.loader.package.loader:loader.py:439 1 failed branches WARNING swh.loader.package.loader:loader.py:442 Failed branches: 0.1.0 __________________________________________ test_jar_2_visits_without_change __________________________________________ swh_storage = requests_mock_datadir = def test_jar_2_visits_without_change(swh_storage, requests_mock_datadir): """With no prior visit, load a gnu project ends up with 1 snapshot """ url = URL loader = JarLoader(swh_storage, url, artifacts=MVN_ARTIFACTS[1]) actual_load_status = loader.load() > assert actual_load_status["status"] == "eventful" E AssertionError: assert 'failed' == 'eventful' E - eventful E + failed tests/test_jar.py:219: AssertionError ------------------------------------------------ Captured stdout call ------------------------------------------------ In get_versions {'time': 1626111425534, 'url': 'https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.1/sprova4j-0.1.1.jar', 'gid': 'al.aldi', 'aid': 'sprova4j', 'filename': 'sprova4j-0.1.1.jar', 'version': '0.1.1'} _______________________________________________ test_tasks_jar_loader ________________________________________________ mocker = swh_scheduler_celery_app = swh_scheduler_celery_worker = swh_config = '/tmp/pytest-of-boris/pytest-236/test_tasks_jar_loader0/loader.yml' def test_tasks_jar_loader( mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config ): mock_load = mocker.patch("swh.loader.package.jar.loader.JarLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.jar.tasks.LoadJar", kwargs=dict(url="https://gnu.org/", artifacts=[]), ) assert res > res.wait() tests/test_tasks.py:18: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/result.py:230: in get on_message=on_message, /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/backends/base.py:711: in wait_for_pending return result.maybe_throw(propagate=propagate, callback=callback) /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/result.py:335: in maybe_throw self.throw(value, self._to_remote_traceback(tb)) /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/result.py:328: in throw self.on_ready.throw(*args, **kwargs) /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/vine/promises.py:234: in throw reraise(type(exc), exc, tb) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tp = value = Task of kind 'swh.loader.package.jar.tasks.LoadJar' never registered, please make sure it's imported. tb = None def reraise(tp, value, tb=None): """Reraise exception.""" if value.__traceback__ is not tb: raise value.with_traceback(tb) > raise value E celery.exceptions.NotRegistered: 'swh.loader.package.jar.tasks.LoadJar' /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/vine/utils.py:30: NotRegistered ----------------------------------------------- Captured stderr setup ------------------------------------------------ [2021-08-30 10:24:07,481: WARNING/MainProcess] /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/kombu/utils/compat.py:93: DeprecationWarning: SelectableGroups dict interface is deprecated. Use select. for ep in importlib_metadata.entry_points().get(namespace, []) [2021-08-30 10:24:07,487: INFO/MainProcess] Connected to memory://guest@localhost// [2021-08-30 10:24:08,506: INFO/MainProcess] Task celery.ping[884dab2c-ed4e-42d1-9dde-d8d43f6d8b08] received [2021-08-30 10:24:08,511: INFO/MainProcess] Task celery.ping[884dab2c-ed4e-42d1-9dde-d8d43f6d8b08] succeeded in 0.0024634290020912886s: 'pong' ------------------------------------------------- Captured log setup ------------------------------------------------- WARNING py.warnings:warnings.py:110 /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/kombu/utils/compat.py:93: DeprecationWarning: SelectableGroups dict interface is deprecated. Use select. for ep in importlib_metadata.entry_points().get(namespace, []) INFO celery.worker.consumer.connection:connection.py:22 Connected to memory://guest@localhost// INFO celery.worker.strategy:strategy.py:155 Task celery.ping[884dab2c-ed4e-42d1-9dde-d8d43f6d8b08] received INFO celery.app.trace:trace.py:131 Task celery.ping[884dab2c-ed4e-42d1-9dde-d8d43f6d8b08] succeeded in 0.0024634290020912886s: 'pong' ------------------------------------------------ Captured stderr call ------------------------------------------------ [2021-08-30 10:24:09,519: ERROR/MainProcess] Received unregistered task of type 'swh.loader.package.jar.tasks.LoadJar'. The message has been ignored and discarded. Did you remember to import the module containing this task? Or maybe you're using relative imports? Please see http://docs.celeryq.org/en/latest/internals/protocol.html for more information. The full contents of the message body was: b'\x93\x90\x82\xa3url\xb0https://gnu.org/\xa9artifacts\x90\x84\xa9callbacks\xc0\xa8errbacks\xc0\xa5chain\xc0\xa5chord\xc0' (71b) Traceback (most recent call last): File "/home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/worker/consumer/consumer.py", line 581, in on_task_received strategy = strategies[type_] KeyError: 'swh.loader.package.jar.tasks.LoadJar' ------------------------------------------------- Captured log call -------------------------------------------------- ERROR celery.worker.consumer.consumer:consumer.py:515 Received unregistered task of type 'swh.loader.package.jar.tasks.LoadJar'. The message has been ignored and discarded. Did you remember to import the module containing this task? Or maybe you're using relative imports? Please see http://docs.celeryq.org/en/latest/internals/protocol.html for more information. The full contents of the message body was: b'\x93\x90\x82\xa3url\xb0https://gnu.org/\xa9artifacts\x90\x84\xa9callbacks\xc0\xa8errbacks\xc0\xa5chain\xc0\xa5chord\xc0' (71b) Traceback (most recent call last): File "/home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/worker/consumer/consumer.py", line 581, in on_task_received strategy = strategies[type_] KeyError: 'swh.loader.package.jar.tasks.LoadJar' _______________________________________ test_tasks_jar_loader_snapshot_append ________________________________________ mocker = swh_scheduler_celery_app = swh_scheduler_celery_worker = swh_config = '/tmp/pytest-of-boris/pytest-236/test_tasks_jar_loader_snapshot0/loader.yml' def test_tasks_jar_loader_snapshot_append( mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config ): mock_load = mocker.patch("swh.loader.package.jar.loader.JarLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.jar.tasks.LoadJar", kwargs=dict(url="https://gnu.org/", artifacts=[], snapshot_append=True), ) assert res > res.wait() tests/test_tasks.py:35: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/result.py:230: in get on_message=on_message, /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/backends/base.py:711: in wait_for_pending return result.maybe_throw(propagate=propagate, callback=callback) /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/result.py:335: in maybe_throw self.throw(value, self._to_remote_traceback(tb)) /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/result.py:328: in throw self.on_ready.throw(*args, **kwargs) /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/vine/promises.py:234: in throw reraise(type(exc), exc, tb) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tp = value = Task of kind 'swh.loader.package.jar.tasks.LoadJar' never registered, please make sure it's imported. tb = None def reraise(tp, value, tb=None): """Reraise exception.""" if value.__traceback__ is not tb: raise value.with_traceback(tb) > raise value E celery.exceptions.NotRegistered: 'swh.loader.package.jar.tasks.LoadJar' /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/vine/utils.py:30: NotRegistered ------------------------------------------------ Captured stderr call ------------------------------------------------ [2021-08-30 10:24:10,524: ERROR/MainProcess] Received unregistered task of type 'swh.loader.package.jar.tasks.LoadJar'. The message has been ignored and discarded. Did you remember to import the module containing this task? Or maybe you're using relative imports? Please see http://docs.celeryq.org/en/latest/internals/protocol.html for more information. The full contents of the message body was: b'\x93\x90\x83\xa3url\xb0https://gnu.org/\xa9artifacts\x90\xafsnapshot_append\xc3\x84\xa9callbacks\xc0\xa8errbacks\xc0\xa5chain\xc0\xa5chord\xc0' (88b) Traceback (most recent call last): File "/home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/worker/consumer/consumer.py", line 581, in on_task_received strategy = strategies[type_] KeyError: 'swh.loader.package.jar.tasks.LoadJar' ------------------------------------------------- Captured log call -------------------------------------------------- ERROR celery.worker.consumer.consumer:consumer.py:515 Received unregistered task of type 'swh.loader.package.jar.tasks.LoadJar'. The message has been ignored and discarded. Did you remember to import the module containing this task? Or maybe you're using relative imports? Please see http://docs.celeryq.org/en/latest/internals/protocol.html for more information. The full contents of the message body was: b'\x93\x90\x83\xa3url\xb0https://gnu.org/\xa9artifacts\x90\xafsnapshot_append\xc3\x84\xa9callbacks\xc0\xa8errbacks\xc0\xa5chain\xc0\xa5chord\xc0' (88b) Traceback (most recent call last): File "/home/boris/.virtualenvs/swh/lib/python3.7/site-packages/celery/worker/consumer/consumer.py", line 581, in on_task_received strategy = strategies[type_] KeyError: 'swh.loader.package.jar.tasks.LoadJar' ================================================== warnings summary ================================================== ../../../../../../.virtualenvs/swh/lib/python3.7/site-packages/kombu/utils/compat.py:93 /home/boris/.virtualenvs/swh/lib/python3.7/site-packages/kombu/utils/compat.py:93: DeprecationWarning: SelectableGroups dict interface is deprecated. Use select. for ep in importlib_metadata.entry_points().get(namespace, []) -- Docs: https://docs.pytest.org/en/stable/warnings.html ============================================== short test summary info =============================================== FAILED tests/test_jar.py::test_jar_visit_with_release_artifact_no_prior_visit - AssertionError: assert {'content': ... FAILED tests/test_jar.py::test_jar_2_visits_without_change - AssertionError: assert 'failed' == 'eventful' FAILED tests/test_tasks.py::test_tasks_jar_loader - celery.exceptions.NotRegistered: 'swh.loader.package.jar.tasks.... FAILED tests/test_tasks.py::test_tasks_jar_loader_snapshot_append - celery.exceptions.NotRegistered: 'swh.loader.pa... ====================================== 4 failed, 1 passed, 1 warning in 10.97s =======================================