Page MenuHomeSoftware Heritage
Files F7124860

D1967.id6641.diff
No OneTemporary
Actions

D1967.id6641.diff
View Options

diff --git a/pytest.ini b/pytest.ini
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,2 +1,6 @@
[pytest]
norecursedirs = docs
+
+markers =
+ db: marks tests as using a db (deselect with '-m "not db"')
+ fs: marks tests as using the filesystem (deselect with '-m "not fs"')
diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py
--- a/swh/loader/core/loader.py
+++ b/swh/loader/core/loader.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 The Software Heritage developers
+# Copyright (C) 2015-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -210,11 +210,12 @@
if not hasattr(self, '__save_data_path'):
year = str(self.visit_date.year)
- origin_url_hash = hashlib.sha1(self.origin['url']).hexdigest()
+ url = self.origin['url'].encode('utf-8')
+ origin_url_hash = hashlib.sha1(url).hexdigest()
- path = os.path.join(
+ path = '%s/sha1:%s/%s/%s' % (
self.config['save_data_path'],
- 'sha1:' + origin_url_hash[0:2],
+ origin_url_hash[0:2],
origin_url_hash,
year,
)
diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py
--- a/swh/loader/core/tests/test_loader.py
+++ b/swh/loader/core/tests/test_loader.py
@@ -1,10 +1,12 @@
-# Copyright (C) 2018 The Software Heritage developers
+# Copyright (C) 2018-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import datetime
+import hashlib
import logging
+import pytest
from swh.model.hashutil import hash_to_bytes
@@ -63,7 +65,9 @@
class DummyBufferedLoader(DummyLoader, BufferedLoader):
- pass
+ def __init__(self, *args, save_data_path=None, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.__save_data_path = save_data_path
class DummyBaseLoaderTest(BaseLoaderTest):
@@ -344,3 +348,24 @@
assert isinstance(loader.log, logging.Logger)
assert loader.log.name == \
'some.logger.name'
+
+
+@pytest.mark.fs
+def test_loader_save_data_path(tmp_path):
+ loader = DummyBufferedLoader('some.logger.name.1')
+ url = 'http://bitbucket.org/something'
+ loader.origin = {
+ 'url': url,
+ }
+ loader.visit_date = datetime.datetime(year=2019, month=10, day=1)
+ loader.config = {
+ 'save_data_path': tmp_path,
+ }
+
+ hash_url = hashlib.sha1(url.encode('utf-8')).hexdigest()
+ expected_save_path = '%s/sha1:%s/%s/2019' % (
+ str(tmp_path), hash_url[0:2], hash_url
+ )
+
+ save_path = loader.get_save_data_path()
+ assert save_path == expected_save_path
diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py
--- a/swh/loader/package/loader.py
+++ b/swh/loader/package/loader.py
@@ -69,7 +69,7 @@
"""
if self.debug:
- self.log.warn('%s Will not pre-clean up temp dir %s' % (
+ self.log.warning('%s Will not pre-clean up temp dir %s' % (
DEBUG_MODE, self.temp_directory
))
return
@@ -114,7 +114,7 @@
self.all_version_data = []
self.latest_timestamp = 0
# Conceled the data into one dictionary to eleminate the need of
- # passing all the paramaters when required in some method
+ # passing all the parameters when required in some method
self.package_details = {
'name': name,
'origin_url': origin_url,
@@ -162,8 +162,8 @@
tarball_request = self._request(tarball_url,
throw_error=False)
if tarball_request.status_code == 404:
- self.log.warn('Tarball url %s returns a 404 error.',
- tarball_url)
+ self.log.warning('Tarball url %s returns a 404 error.',
+ tarball_url)
self._visit_status = 'partial'
# FIX ME: Do we need to mark it `partial` here
continue
@@ -468,7 +468,7 @@
"""
if self.debug:
- self.log.warn('%s Will not clean up temp dir %s' % (
+ self.log.warning('%s Will not clean up temp dir %s' % (
DEBUG_MODE, self.temp_directory
))
return

File Metadata

Mime Type
text/plain
Expires
Dec 21 2024, 9:56 PM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3227407

Event Timeline

About · Developer information · Legal