diff --git a/swh/lister/gnu/lister.py b/swh/lister/gnu/lister.py --- a/swh/lister/gnu/lister.py +++ b/swh/lister/gnu/lister.py @@ -103,7 +103,7 @@ directory['name'], repo['name']) package_tarballs = find_tarballs( - repo['contents'], package_url) + repo['contents'], package_url) if package_tarballs != []: repo_details = { 'name': repo['name'], @@ -125,7 +125,7 @@ 'full_name': repo['name'], 'html_url': repo['url'], 'origin_url': repo['url'], - 'time_last_updated': repo['time_modified'], + 'time_last_updated': int(repo['time_modified']), 'origin_type': 'tar', } @@ -137,44 +137,47 @@ def find_tarballs(package_file_structure, url): - ''' - Recursively lists all the tarball present in the folder and - subfolders for a particular package url. + '''Recursively lists tarballs present in the folder and subfolders for a + particular package url. Args - package_file_structure : File structure of the package root directory - url : URL of the corresponding package + package_file_structure: File structure of the package root directory + url: URL of the corresponding package Returns - List of all the tarball urls and the last their time of update - example- - For a package called 3dldf + List of tarball urls and their associated metadata (time, length). + For example: [ {'archive': 'https://ftp.gnu.org/gnu/3dldf/3DLDF-1.1.3.tar.gz', - 'date': '1071002600'} + 'time': 1071002600, + 'length': 543}, {'archive': 'https://ftp.gnu.org/gnu/3dldf/3DLDF-1.1.4.tar.gz', - 'date': '1071078759'} + 'time': 1071078759, + 'length': 456}, {'archive': 'https://ftp.gnu.org/gnu/3dldf/3DLDF-1.1.5.1.tar.gz', - 'date': '1074278633'} + 'time': 1074278633, + 'length': 251}, ... ] + ''' tarballs = [] for single_file in package_file_structure: - file_type = single_file['type'] - file_name = single_file['name'] - if file_type == 'file': - if file_extension_check(file_name): - tarballs .append({ - "archive": url + file_name, - "date": single_file['time'] + filetype = single_file['type'] + filename = single_file['name'] + if filetype == 'file': + if file_extension_check(filename): + tarballs.append({ + 'archive': url + filename, + 'time': int(single_file['time']), + 'length': int(single_file['size']), }) # It will recursively check for tarballs in all sub-folders - elif file_type == 'directory': + elif filetype == 'directory': tarballs_in_dir = find_tarballs( - single_file['contents'], - url + file_name + '/') + single_file['contents'], + url + filename + '/') tarballs.extend(tarballs_in_dir) return tarballs diff --git a/swh/lister/gnu/tests/test_lister.py b/swh/lister/gnu/tests/test_lister.py --- a/swh/lister/gnu/tests/test_lister.py +++ b/swh/lister/gnu/tests/test_lister.py @@ -17,20 +17,39 @@ assert False +def test_find_tarballs_small_sample(): + expected_tarballs = [ + { + 'archive': '/root/artanis/artanis-0.2.1.tar.bz2', + 'time': 1495205979, + 'length': 424081, + }, + { + 'archive': '/root/xboard/winboard/winboard-4_0_0-src.zip', # noqa + 'time': 898422900, + 'length': 1514448 + }, + { + 'archive': '/root/xboard/xboard-3.6.2.tar.gz', # noqa + 'time': 869814000, + 'length': 450164, + }, + { + 'archive': '/root/xboard/xboard-4.0.0.tar.gz', # noqa + 'time': 898422900, + 'length': 514951, + }, + ] + + file_structure = json.load(open('swh/lister/gnu/tests/tree.min.json')) + actual_tarballs = find_tarballs(file_structure, '/root/') + assert actual_tarballs == expected_tarballs + + def test_find_tarballs(): - f = open('swh/lister/gnu/tests/find_tarballs_output.json') - expected_list_of_all_tarballs = json.load(f) - - f = open('swh/lister/gnu/tests/file_structure.json') - file_structure = json.load(f) - list_of_all_tarballs = [] - list_of_all_tarballs.extend( - find_tarballs(file_structure[0]['contents'], - "https://ftp.gnu.org/gnu/artanis/")) - list_of_all_tarballs.extend( - find_tarballs(file_structure[1]['contents'], - "https://ftp.gnu.org/old-gnu/xboard/")) - assert list_of_all_tarballs == expected_list_of_all_tarballs + file_structure = json.load(open('swh/lister/gnu/tests/tree.json')) + actual_tarballs = find_tarballs(file_structure, '/root/') + assert len(actual_tarballs) == 42 + 3 # tar + zip def test_file_extension_check(): diff --git a/swh/lister/gnu/tests/file_structure.json b/swh/lister/gnu/tests/tree.json rename from swh/lister/gnu/tests/file_structure.json rename to swh/lister/gnu/tests/tree.json --- a/swh/lister/gnu/tests/file_structure.json +++ b/swh/lister/gnu/tests/tree.json @@ -70,4 +70,4 @@ {"type":"file","name":"xboard-4.2.6.tar.gz","size":1057625,"time":"1012641715"}, {"type":"file","name":"xboard-4.2.7.tar.gz","size":1318110,"time":"1070057764"} ]} -] \ No newline at end of file +] diff --git a/swh/lister/gnu/tests/tree.min.json b/swh/lister/gnu/tests/tree.min.json new file mode 100644 --- /dev/null +++ b/swh/lister/gnu/tests/tree.min.json @@ -0,0 +1,16 @@ +[ + {"type":"directory","name":"artanis","size":4096,"time":"1546205705","contents":[ + {"type":"file","name":"artanis-0.2.1.tar.bz2","size":424081,"time":"1495205979"}, + {"type":"file","name":"artanis-0.2.1.tar.bz2.sig","size":833,"time":"1495205982"} + ]}, + {"type":"directory","name":"xboard","size":4096,"time":"1254860068","contents":[ + {"type":"directory","name":"winboard","size":4096,"time":"1181795103","contents":[ + {"type":"file","name":"README","size":107,"time":"1070058107"}, + {"type":"file","name":"README.sig","size":65,"time":"1070058115"}, + {"type":"file","name":"winboard-4_0_0-src.zip","size":1514448,"time":"898422900"}, + {"type":"file","name":"winboard-4_0_0.exe","size":1652037,"time":"898422900"} + ]}, + {"type":"file","name":"xboard-3.6.2.tar.gz","size":450164,"time":"869814000"}, + {"type":"file","name":"xboard-4.0.0.tar.gz","size":514951,"time":"898422900"} + ]} +] diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,9 @@ .[testing] pytest-cov commands = - pytest --cov=swh --cov-branch {posargs} + pytest --cov={envsitepackagesdir}/swh/lister/ --cov-branch \ + {envsitepackagesdir}/swh/lister/ {posargs} + [testenv:flake8] skip_install = true