Page MenuHomeSoftware Heritage
Paste P1280

docker run says no to naive datetime ¯\_(ツ)_/¯
ActivePublic

Authored by ardumont on Feb 8 2022, 11:31 AM.
swh-lister_1 | [2022-02-08 10:29:25,384: INFO/MainProcess] Task swh.lister.maven.tasks.FullMavenLister[95fa1dcb-6fea-41b7-82e3-e07162d45f2d] received
swh-lister_1 | [2022-02-08 10:29:45,424: INFO/ForkPoolWorker-1] Downloading text index from http://swh-lister-maven-nginx/export.fld.
swh-lister_1 | [2022-02-08 10:30:05,455: ERROR/ForkPoolWorker-1] Task swh.lister.maven.tasks.FullMavenLister[95fa1dcb-6fea-41b7-82e3-e07162d45f2d] raised unexpected: ValueError("can not serialize 'datetime.datetime' object where tzinfo=None")
swh-lister_1 | Traceback (most recent call last):
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/celery/app/trace.py", line 450, in trace_task
swh-lister_1 | R = retval = fun(*args, **kwargs)
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/scheduler/task.py", line 55, in __call__
swh-lister_1 | result = super().__call__(*args, **kwargs)
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/celery/app/trace.py", line 731, in __protected_call__
swh-lister_1 | return self.run(*args, **kwargs)
swh-lister_1 | File "/src/swh-lister/swh/lister/maven/tasks.py", line 16, in list_maven_full
swh-lister_1 | return lister.run().dict()
swh-lister_1 | File "/src/swh-lister/swh/lister/pattern.py", line 130, in run
swh-lister_1 | full_stats.origins += self.send_origins(origins)
swh-lister_1 | File "/src/swh-lister/swh/lister/pattern.py", line 234, in send_origins
swh-lister_1 | ret = self.scheduler.record_listed_origins(batch_origins)
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/__init__.py", line 181, in meth_
swh-lister_1 | return self.post(meth._endpoint_path, post_data)
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/__init__.py", line 262, in post
swh-lister_1 | data = self._encode_data(data)
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/__init__.py", line 281, in _encode_data
swh-lister_1 | return encode_data(data, extra_encoders=self.extra_type_encoders)
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/serializers.py", line 126, in encode_data_client
swh-lister_1 | return msgpack_dumps(data, extra_encoders=extra_encoders)
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/serializers.py", line 275, in msgpack_dumps
swh-lister_1 | default=encode_types,
swh-lister_1 | File "/srv/softwareheritage/venv/lib/python3.7/site-packages/msgpack/__init__.py", line 35, in packb
swh-lister_1 | return Packer(**kwargs).pack(o)
swh-lister_1 | File "msgpack/_packer.pyx", line 294, in msgpack._cmsgpack.Packer.pack
swh-lister_1 | File "msgpack/_packer.pyx", line 300, in msgpack._cmsgpack.Packer.pack
swh-lister_1 | File "msgpack/_packer.pyx", line 297, in msgpack._cmsgpack.Packer.pack
swh-lister_1 | File "msgpack/_packer.pyx", line 231, in msgpack._cmsgpack.Packer._pack
swh-lister_1 | File "msgpack/_packer.pyx", line 264, in msgpack._cmsgpack.Packer._pack
swh-lister_1 | File "msgpack/_packer.pyx", line 231, in msgpack._cmsgpack.Packer._pack
swh-lister_1 | File "msgpack/_packer.pyx", line 231, in msgpack._cmsgpack.Packer._pack
swh-lister_1 | File "msgpack/_packer.pyx", line 289, in msgpack._cmsgpack.Packer._pack
swh-lister_1 | ValueError: can not serialize 'datetime.datetime' object where tzinfo=None

Event Timeline

ardumont changed the title of this paste from untitled to docker run says no to datetime ¯\_(ツ)_/¯.
(Pdb++) origins_[0]
ListedOrigin(lister_id=UUID('31fd49a6-396e-473e-8522-ed105b48b5d9'), url='https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/sprova4j-0.1.0-sources.jar', visit_type='maven', extra_loader_arguments={'artifacts': [{'time': '2021-07-12T17:06:59+00:00', 'gid': 'al.aldi', 'aid': 'sprova4j', 'version': '0.1.0', 'base_url': 'https://repo1.maven.org/maven2/'}]}, last_update=datetime.datetime(2021, 7, 12, 17, 6, 59), enabled=True, first_seen=None, last_seen=None)
(Pdb++) origins_[0].last_update
datetime.datetime(2021, 7, 12, 17, 6, 59)
(Pdb++) self.send_origins(origins_)
*** ValueError: can not serialize 'datetime.datetime' object where tzinfo=None
Traceback (most recent call last):
  File "/src/swh-lister/swh/lister/pattern.py", line 236, in send_origins
    ret = self.scheduler.record_listed_origins(batch_origins)
  File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/__init__.py", line 181, in meth_
    return self.post(meth._endpoint_path, post_data)
  File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/__init__.py", line 262, in post
    data = self._encode_data(data)
  File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/__init__.py", line 281, in _encode_data
    return encode_data(data, extra_encoders=self.extra_type_encoders)
  File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/serializers.py", line 126, in encode_data_client
    return msgpack_dumps(data, extra_encoders=extra_encoders)
  File "/srv/softwareheritage/venv/lib/python3.7/site-packages/swh/core/api/serializers.py", line 275, in msgpack_dumps
    default=encode_types,
  File "/srv/softwareheritage/venv/lib/python3.7/site-packages/msgpack/__init__.py", line 35, in packb
    return Packer(**kwargs).pack(o)
  File "msgpack/_packer.pyx", line 294, in msgpack._cmsgpack.Packer.pack
  File "msgpack/_packer.pyx", line 300, in msgpack._cmsgpack.Packer.pack
  File "msgpack/_packer.pyx", line 297, in msgpack._cmsgpack.Packer.pack
  File "msgpack/_packer.pyx", line 231, in msgpack._cmsgpack.Packer._pack
  File "msgpack/_packer.pyx", line 264, in msgpack._cmsgpack.Packer._pack
  File "msgpack/_packer.pyx", line 231, in msgpack._cmsgpack.Packer._pack
  File "msgpack/_packer.pyx", line 231, in msgpack._cmsgpack.Packer._pack
  File "msgpack/_packer.pyx", line 289, in msgpack._cmsgpack.Packer._pack
(Pdb++) origin = origins_[0]
(Pdb++) origin
ListedOrigin(lister_id=UUID('31fd49a6-396e-473e-8522-ed105b48b5d9'), url='https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/sprova4j-0.1.0-sources.jar', visit_type='maven', extra_loader_arguments={'artifacts': [{'time': '2021-07-12T17:06:59+00:00', 'gid': 'al.aldi', 'aid': 'sprova4j', 'version': '0.1.0', 'base_url': 'https://repo1.maven.org/maven2/'}]}, last_update=datetime.datetime(2021, 7, 12, 17, 6, 59), enabled=True, first_seen=None, last_seen=None)
(Pdb++) last_update = origin.last_update
(Pdb++) last_update
datetime.datetime(2021, 7, 12, 17, 6, 59)
(Pdb++) last_update.replace
last_update.replace
(Pdb++) last_update.replace(tzinfo=timezone.utc)
*** NameError: name 'timezone' is not defined
(Pdb++) from datetime import timezone
(Pdb++) last_update.replace(tzinfo=timezone.utc)
datetime.datetime(2021, 7, 12, 17, 6, 59, tzinfo=datetime.timezone.utc)
(Pdb++) origin=attr.evolve(origin, last_update=last_update.replace(tzinfo=timezone.utc))
*** NameError: name 'attr' is not defined
(Pdb++) import attr
(Pdb++) origin=attr.evolve(origin, last_update=last_update.replace(tzinfo=timezone.utc))
(Pdb++) origin
ListedOrigin(lister_id=UUID('31fd49a6-396e-473e-8522-ed105b48b5d9'), url='https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/sprova4j-0.1.0-sources.jar', visit_type='maven', extra_loader_arguments={'artifacts': [{'time': '2021-07-12T17:06:59+00:00', 'gid': 'al.aldi', 'aid': 'sprova4j', 'version': '0.1.0', 'base_url': 'https://repo1.maven.org/maven2/'}]}, last_update=datetime.datetime(2021, 7, 12, 17, 6, 59, tzinfo=datetime.timezone.utc), enabled=True, first_seen=None, last_seen=None)
(Pdb++) self.send_origins([origin])
1

So we need to add the tzinfo to those datetime.

ardumont changed the title of this paste from docker run says no to datetime ¯\_(ツ)_/¯ to docker run says no to naive datetime ¯\_(ツ)_/¯.Feb 9 2022, 5:00 PM