diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -7,6 +7,7 @@ import bz2 from collections import defaultdict from dataclasses import dataclass, field +from email.utils import parsedate_to_datetime import gzip from itertools import product import logging @@ -134,6 +135,10 @@ response = requests.get(url, stream=True) logging.debug("Fetched URL: %s, status code: %s", url, response.status_code) if response.status_code == 200: + last_modified = response.headers.get("Last-Modified") + self.last_sources_update = ( + parsedate_to_datetime(last_modified) if last_modified else None + ) decompressor = decompressors.get(compression) if decompressor: data = decompressor(response.raw).readlines() @@ -224,6 +229,7 @@ url=origin_url, visit_type="deb", extra_loader_arguments={"packages": {}}, + last_update=self.last_sources_update, ) # origin will be yielded at the end of that method origins_to_send[origin_url] = self.listed_origins[origin_url] diff --git a/swh/lister/debian/tests/test_lister.py b/swh/lister/debian/tests/test_lister.py --- a/swh/lister/debian/tests/test_lister.py +++ b/swh/lister/debian/tests/test_lister.py @@ -4,6 +4,7 @@ # See top-level LICENSE file for more information from collections import defaultdict +from email.utils import formatdate import os from pathlib import Path from typing import Dict, List, Set, Tuple @@ -78,7 +79,11 @@ if compression: requests_mock.get(idx_url, status_code=404) else: - requests_mock.get(idx_url, text=sources) + requests_mock.get( + idx_url, + text=sources, + headers={"Last-Modified": formatdate(usegmt=True)}, + ) for idx_url, _ in lister.debian_index_urls(suite, _components[1]): requests_mock.get(idx_url, status_code=404) @@ -122,6 +127,7 @@ ] assert filtered_origins + assert filtered_origins[0].last_update is not None packages = filtered_origins[0].extra_loader_arguments["packages"] # check the version info are available assert package_version_key in packages