diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 380c658..69b3349 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,46 +1,40 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.4.0
hooks:
- id: trailing-whitespace
- id: flake8
- id: check-json
- id: check-yaml
- repo: https://github.com/codespell-project/codespell
rev: v1.16.0
hooks:
- id: codespell
- repo: local
hooks:
- id: mypy
name: mypy
entry: mypy
args: [swh]
pass_filenames: false
language: system
types: [python]
+- repo: https://github.com/python/black
+ rev: 19.10b0
+ hooks:
+ - id: black
+
# unfortunately, we are far from being able to enable this...
# - repo: https://github.com/PyCQA/pydocstyle.git
# rev: 4.0.0
# hooks:
# - id: pydocstyle
# name: pydocstyle
# description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions.
# entry: pydocstyle --convention=google
# language: python
# types: [python]
-# black requires py3.6+
-#- repo: https://github.com/python/black
-# rev: 19.3b0
-# hooks:
-# - id: black
-# language_version: python3
-#- repo: https://github.com/asottile/blacken-docs
-# rev: v1.0.0-1
-# hooks:
-# - id: blacken-docs
-# additional_dependencies: [black==19.3b0]
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..8d79b7e
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,6 @@
+[flake8]
+# E203: whitespaces before ':' <https://github.com/psf/black/issues/315>
+# E231: missing whitespace after ','
+# W503: line break before binary operator <https://github.com/psf/black/issues/52>
+ignore = E203,E231,W503
+max-line-length = 88
diff --git a/setup.py b/setup.py
index 78ab3c1..57820ec 100755
--- a/setup.py
+++ b/setup.py
@@ -1,72 +1,72 @@
#!/usr/bin/env python3
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from setuptools import setup, find_packages
from os import path
from io import open
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
-with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
+with open(path.join(here, "README.rst"), encoding="utf-8") as f:
long_description = f.read()
def parse_requirements(name=None):
if name:
- reqf = 'requirements-%s.txt' % name
+ reqf = "requirements-%s.txt" % name
else:
- reqf = 'requirements.txt'
+ reqf = "requirements.txt"
requirements = []
if not path.exists(reqf):
return requirements
with open(reqf) as f:
for line in f.readlines():
line = line.strip()
- if not line or line.startswith('#'):
+ if not line or line.startswith("#"):
continue
requirements.append(line)
return requirements
# Edit this part to match your module.
# Full sample:
# https://forge.softwareheritage.org/diffusion/DCORE/browse/master/setup.py
setup(
- name='swh.scanner',
- description='Software Heritage code scanner',
+ name="swh.scanner",
+ description="Software Heritage code scanner",
long_description=long_description,
- long_description_content_type='text/x-rst',
- author='Software Heritage developers',
- author_email='swh-devel@inria.fr',
- url='https://forge.softwareheritage.org/diffusion/DTSCN/',
+ long_description_content_type="text/x-rst",
+ author="Software Heritage developers",
+ author_email="swh-devel@inria.fr",
+ url="https://forge.softwareheritage.org/diffusion/DTSCN/",
packages=find_packages(), # packages's modules
- install_requires=parse_requirements() + parse_requirements('swh'),
- tests_require=parse_requirements('test'),
- setup_requires=['vcversioner'],
- extras_require={'testing': parse_requirements('test')},
+ install_requires=parse_requirements() + parse_requirements("swh"),
+ tests_require=parse_requirements("test"),
+ setup_requires=["vcversioner"],
+ extras_require={"testing": parse_requirements("test")},
vcversioner={},
include_package_data=True,
- entry_points='''
+ entry_points="""
[swh.cli.subcommands]
scanner=swh.scanner.cli:scanner
- ''',
+ """,
classifiers=[
"Programming Language :: Python :: 3",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Operating System :: OS Independent",
"Development Status :: 3 - Alpha",
],
project_urls={
- 'Bug Reports': 'https://forge.softwareheritage.org/maniphest',
- 'Funding': 'https://www.softwareheritage.org/donate',
- 'Source': 'https://forge.softwareheritage.org/source/swh-scanner',
+ "Bug Reports": "https://forge.softwareheritage.org/maniphest",
+ "Funding": "https://www.softwareheritage.org/donate",
+ "Source": "https://forge.softwareheritage.org/source/swh-scanner",
},
)
diff --git a/swh/scanner/cli.py b/swh/scanner/cli.py
index 82d7ffa..eb22201 100644
--- a/swh/scanner/cli.py
+++ b/swh/scanner/cli.py
@@ -1,56 +1,62 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import click
import asyncio
from pathlib import PosixPath
from .scanner import run
from .model import Tree
from swh.core.cli import CONTEXT_SETTINGS
-@click.group(name='scanner', context_settings=CONTEXT_SETTINGS)
+@click.group(name="scanner", context_settings=CONTEXT_SETTINGS)
@click.pass_context
def scanner(ctx):
- '''Software Heritage Scanner tools.'''
+ """Software Heritage Scanner tools."""
pass
def parse_url(url):
- if not url.startswith('https://'):
- url = 'https://' + url
- if not url.endswith('/'):
- url += '/'
+ if not url.startswith("https://"):
+ url = "https://" + url
+ if not url.endswith("/"):
+ url += "/"
return url
-@scanner.command(name='scan')
-@click.argument('path', required=True, type=click.Path(exists=True))
-@click.option('-u', '--api-url',
- default='https://archive.softwareheritage.org/api/1',
- metavar='API_URL', show_default=True,
- help="url for the api request")
-@click.option('-f', '--format',
- type=click.Choice(['text', 'json', 'sunburst'],
- case_sensitive=False),
- default='text',
- help="select the output format")
+@scanner.command(name="scan")
+@click.argument("path", required=True, type=click.Path(exists=True))
+@click.option(
+ "-u",
+ "--api-url",
+ default="https://archive.softwareheritage.org/api/1",
+ metavar="API_URL",
+ show_default=True,
+ help="url for the api request",
+)
+@click.option(
+ "-f",
+ "--format",
+ type=click.Choice(["text", "json", "sunburst"], case_sensitive=False),
+ default="text",
+ help="select the output format",
+)
@click.pass_context
def scan(ctx, path, api_url, format):
"""Scan a source code project to discover files and directories already
present in the archive"""
api_url = parse_url(api_url)
source_tree = Tree(PosixPath(path))
loop = asyncio.get_event_loop()
loop.run_until_complete(run(path, api_url, source_tree))
source_tree.show(format)
-if __name__ == '__main__':
+if __name__ == "__main__":
scan()
diff --git a/swh/scanner/exceptions.py b/swh/scanner/exceptions.py
index e9e482b..2618950 100644
--- a/swh/scanner/exceptions.py
+++ b/swh/scanner/exceptions.py
@@ -1,18 +1,18 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
class InvalidObjectType(TypeError):
pass
class APIError(Exception):
def __str__(self):
return '"%s"' % self.args
def error_response(reason: str, status_code: int, api_url: str):
- error_msg = f'{status_code} {reason}: \'{api_url}\''
+ error_msg = f"{status_code} {reason}: '{api_url}'"
raise APIError(error_msg)
diff --git a/swh/scanner/model.py b/swh/scanner/model.py
index 7d647a8..2e0aca2 100644
--- a/swh/scanner/model.py
+++ b/swh/scanner/model.py
@@ -1,181 +1,179 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from __future__ import annotations
import sys
import json
from pathlib import PosixPath
from typing import Any, Dict, Tuple
from enum import Enum
from .plot import sunburst
from .exceptions import InvalidObjectType
-from swh.model.identifiers import (
- DIRECTORY, CONTENT
-)
+from swh.model.identifiers import DIRECTORY, CONTENT
class Color(Enum):
- blue = '\033[94m'
- green = '\033[92m'
- red = '\033[91m'
- end = '\033[0m'
+ blue = "\033[94m"
+ green = "\033[92m"
+ red = "\033[91m"
+ end = "\033[0m"
def colorize(text: str, color: Color):
return color.value + text + Color.end.value
class Tree:
"""Representation of a file system structure
"""
+
def __init__(self, path: PosixPath, father: Tree = None):
self.father = father
self.path = path
self.otype = DIRECTORY if path.is_dir() else CONTENT
- self.pid = ''
+ self.pid = ""
self.children: Dict[PosixPath, Tree] = {}
def addNode(self, path: PosixPath, pid: str = None) -> None:
"""Recursively add a new path.
"""
relative_path = path.relative_to(self.path)
- if relative_path == PosixPath('.'):
+ if relative_path == PosixPath("."):
if pid is not None:
self.pid = pid
return
new_path = self.path.joinpath(relative_path.parts[0])
if new_path not in self.children:
self.children[new_path] = Tree(new_path, self)
self.children[new_path].addNode(path, pid)
def show(self, format) -> None:
"""Show tree in different formats"""
- if format == 'json':
+ if format == "json":
print(json.dumps(self.getTree(), indent=4, sort_keys=True))
- elif format == 'text':
+ elif format == "text":
isatty = sys.stdout.isatty()
- print(colorize(str(self.path), Color.blue) if isatty
- else str(self.path))
+ print(colorize(str(self.path), Color.blue) if isatty else str(self.path))
self.printChildren(isatty)
- elif format == 'sunburst':
+ elif format == "sunburst":
root = self.path
directories = self.getDirectoriesInfo(root)
sunburst(directories, root)
def printChildren(self, isatty: bool, inc: int = 1) -> None:
for path, node in self.children.items():
self.printNode(node, isatty, inc)
if node.children:
- node.printChildren(isatty, inc+1)
+ node.printChildren(isatty, inc + 1)
def printNode(self, node: Any, isatty: bool, inc: int) -> None:
rel_path = str(node.path.relative_to(self.path))
- begin = '│ ' * inc
- end = '/' if node.otype == DIRECTORY else ''
+ begin = "│ " * inc
+ end = "/" if node.otype == DIRECTORY else ""
if isatty:
if not node.pid:
rel_path = colorize(rel_path, Color.red)
elif node.otype == DIRECTORY:
rel_path = colorize(rel_path, Color.blue)
elif node.otype == CONTENT:
rel_path = colorize(rel_path, Color.green)
- print(f'{begin}{rel_path}{end}')
+ print(f"{begin}{rel_path}{end}")
def getTree(self):
"""Walk through the tree to discover content or directory that have
a persistent identifier. If a persistent identifier is found it saves
the path with the relative PID.
Returns:
child_tree: the tree with the content/directory found
"""
child_tree = {}
for path, child_node in self.children.items():
rel_path = str(child_node.path.relative_to(self.path))
if child_node.pid:
child_tree[rel_path] = child_node.pid
else:
next_tree = child_node.getTree()
if next_tree:
child_tree[rel_path] = next_tree
return child_tree
def __getSubDirsInfo(self, root, directories):
"""Fills the directories given in input with the contents information
stored inside the directory child, only if they have contents.
"""
for path, child_node in self.children.items():
if child_node.otype == DIRECTORY:
rel_path = path.relative_to(root)
contents_info = child_node.count_contents()
# checks the first element of the tuple
# (the number of contents in a directory)
# if it is equal to zero it means that there are no contents
# in that directory.
if not contents_info[0] == 0:
directories[rel_path] = contents_info
if child_node.has_dirs():
child_node.__getSubDirsInfo(root, directories)
- def getDirectoriesInfo(self, root: PosixPath
- ) -> Dict[PosixPath, Tuple[int, int]]:
+ def getDirectoriesInfo(self, root: PosixPath) -> Dict[PosixPath, Tuple[int, int]]:
"""Get information about all directories under the given root.
Returns:
A dictionary with a directory path as key and the relative
contents information (the result of count_contents) as values.
"""
directories = {root: self.count_contents()}
self.__getSubDirsInfo(root, directories)
return directories
def count_contents(self) -> Tuple[int, int]:
"""Count how many contents are present inside a directory.
If a directory has a pid returns as it has all the contents.
Returns:
A tuple with the total number of the contents and the number
of contents known (the ones that have a persistent identifier).
"""
contents = 0
discovered = 0
if not self.otype == DIRECTORY:
- raise InvalidObjectType('Can\'t calculate contents of the '
- 'object type: %s' % self.otype)
+ raise InvalidObjectType(
+ "Can't calculate contents of the " "object type: %s" % self.otype
+ )
if self.pid:
# to identify a directory with all files/directories present
return (1, 1)
else:
for _, child_node in self.children.items():
if child_node.otype == CONTENT:
contents += 1
if child_node.pid:
discovered += 1
return (contents, discovered)
def has_dirs(self) -> bool:
"""Checks if node has directories
"""
for _, child_node in self.children.items():
if child_node.otype == DIRECTORY:
return True
return False
diff --git a/swh/scanner/plot.py b/swh/scanner/plot.py
index ae4d467..18bb470 100644
--- a/swh/scanner/plot.py
+++ b/swh/scanner/plot.py
@@ -1,264 +1,272 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""
The purpose of this module is to display and to interact with the result of the
scanner contained in the model.
The `sunburst` function generates a navigable sunburst chart from the
directories information retrieved from the model. The chart displays for
each directory the total number of files and the percentage of file known.
The size of the directory is defined by the total number of contents whereas
the color gradient is generated relying on the percentage of contents known.
"""
from typing import List, Dict, Tuple
from pathlib import PosixPath
from plotly.offline import offline # type: ignore
import plotly.graph_objects as go # type: ignore
import pandas as pd # type: ignore
import numpy as np # type: ignore
def build_hierarchical_df(
- dirs_dataframe: pd.DataFrame, levels: List[str],
- metrics_columns: List[str], root_name: str) -> pd.DataFrame:
+ dirs_dataframe: pd.DataFrame,
+ levels: List[str],
+ metrics_columns: List[str],
+ root_name: str,
+) -> pd.DataFrame:
"""
Build a hierarchy of levels for Sunburst or Treemap charts.
For each directory the new dataframe will have the following
information:
id: the directory name
parent: the parent directory of id
contents: the total number of contents of the directory id and
the relative subdirectories
known: the percentage of contents known relative to computed
'contents'
Example:
Given the following dataframe:
.. code-block:: none
lev0 lev1 contents known
'' '' 20 2 //root
kernel kernel/subdirker 5 0
telnet telnet/subdirtel 10 4
The output hierarchical dataframe will be like the following:
.. code-block:: none
id parent contents known
20 10.00
kernel/subdirker kernel 5 0.00
telnet/subdirtel telnet 10 40.00
total 20 10.00
kernel total 5 0.00
telnet total 10 40.00
total 35 17.14
To create the hierarchical dataframe we need to iterate through
the dataframe given in input relying on the number of levels.
Based on the previous example we have to do two iterations:
iteration 1
The generated dataframe 'df_tree' will be:
.. code-block:: none
id parent contents known
20 10.0
kernel/subdirker kernel 5 0.0
telnet/subdirtel telnet 10 40.0
iteration 2
The generated dataframe 'df_tree' will be:
.. code-block:: none
id parent contents known
total 20 10.0
kernel total 5 0.0
telnet total 10 40.0
Note that since we have reached the last level, the parent given
to the directory id is the directory root.
The 'total' row il computed by adding the number of contents of the
dataframe given in input and the average of the contents known on
the total number of contents.
"""
- def compute_known_percentage(contents: pd.Series, known: pd.Series
- ) -> pd.Series:
+
+ def compute_known_percentage(contents: pd.Series, known: pd.Series) -> pd.Series:
"""This function compute the percentage of known contents and generate
the new known column with the percentage values.
It also assures that if there is no contents inside a directory
the percentage is zero
"""
known_values = []
for idx, content_val in enumerate(contents):
if content_val == 0:
known_values.append(0)
else:
percentage = known[idx] / contents[idx] * 100
known_values.append(percentage)
return pd.Series(np.array(known_values))
- complete_df = pd.DataFrame(columns=['id', 'parent', 'contents', 'known'])
+ complete_df = pd.DataFrame(columns=["id", "parent", "contents", "known"])
# revert the level order to start from the deepest
levels = [level for level in reversed(levels)]
contents_col = metrics_columns[0]
known_col = metrics_columns[1]
df_tree_list = []
for i, level in enumerate(levels):
- df_tree = pd.DataFrame(columns=['id', 'parent', 'contents', 'known'])
+ df_tree = pd.DataFrame(columns=["id", "parent", "contents", "known"])
dfg = dirs_dataframe.groupby(levels[i:]).sum()
dfg = dfg.reset_index()
- df_tree['id'] = dfg[level].copy()
+ df_tree["id"] = dfg[level].copy()
if i < len(levels) - 1:
# copy the parent directories (one level above)
- df_tree['parent'] = dfg[levels[i+1]].copy()
+ df_tree["parent"] = dfg[levels[i + 1]].copy()
else:
# last level reached
- df_tree['parent'] = root_name
+ df_tree["parent"] = root_name
# copy the contents column
- df_tree['contents'] = dfg[contents_col]
+ df_tree["contents"] = dfg[contents_col]
# compute the percentage relative to the contents
- df_tree['known'] = compute_known_percentage(
- dfg[contents_col], dfg[known_col])
+ df_tree["known"] = compute_known_percentage(dfg[contents_col], dfg[known_col])
df_tree_list.append(df_tree)
complete_df = complete_df.append(df_tree_list, ignore_index=True)
# create the main parent
total_contents = dirs_dataframe[contents_col].sum()
total_known = dirs_dataframe[known_col].sum()
total_avg = total_known / total_contents * 100
- total = pd.Series(dict(id=root_name, parent='',
- contents=total_contents,
- known=total_avg))
+ total = pd.Series(
+ dict(id=root_name, parent="", contents=total_contents, known=total_avg)
+ )
complete_df = complete_df.append(total, ignore_index=True)
return complete_df
def compute_max_depth(dirs_path: List[PosixPath], root: PosixPath) -> int:
"""Compute the maximum depth level of the given directory paths.
Example: for `var/log/kernel/` the depth level is 3
"""
max_depth = 0
for dir_path in dirs_path:
if dir_path == root:
continue
dir_depth = len(dir_path.parts)
if dir_depth > max_depth:
max_depth = dir_depth
return max_depth
-def generate_df_from_dirs(dirs: Dict[PosixPath, Tuple[int, int]],
- columns: List[str], root: PosixPath, max_depth: int
- ) -> pd.DataFrame:
+def generate_df_from_dirs(
+ dirs: Dict[PosixPath, Tuple[int, int]],
+ columns: List[str],
+ root: PosixPath,
+ max_depth: int,
+) -> pd.DataFrame:
"""Generate a dataframe from the directories given in input.
Example:
given the following directories as input
.. code-block:: python
dirs = {
'/var/log/': (23, 2),
'/var/log/kernel': (5, 0),
'/var/log/telnet': (10, 3)
}
The generated dataframe will be:
.. code-block:: none
lev0 lev1 lev2 contents known
'var' 'var/log' '' 23 2
'var' 'var/log' 'var/log/kernel' 5 0
'var' 'var/log' 'var/log/telnet' 10 3
"""
+
def get_parents(path: PosixPath):
- parts = path.parts[1:] if path.parts[0] == '/' else path.parts
+ parts = path.parts[1:] if path.parts[0] == "/" else path.parts
- for i in range(1, len(parts)+1):
- yield '/'.join(parts[0:i])
+ for i in range(1, len(parts) + 1):
+ yield "/".join(parts[0:i])
def get_dirs_array():
for dir_path, contents_info in dirs.items():
empty_lvl = max_depth - len(dir_path.parts)
if dir_path == root:
# ignore the root but store contents information
- yield ['']*(max_depth) + list(contents_info)
+ yield [""] * (max_depth) + list(contents_info)
else:
- yield list(get_parents(dir_path)) + \
- ['']*empty_lvl + \
- list(contents_info)
+ yield list(get_parents(dir_path)) + [""] * empty_lvl + list(
+ contents_info
+ )
- df = pd.DataFrame(np.array(
- [dir_array for dir_array in get_dirs_array()]), columns=columns)
+ df = pd.DataFrame(
+ np.array([dir_array for dir_array in get_dirs_array()]), columns=columns
+ )
- df['contents'] = pd.to_numeric(df['contents'])
- df['known'] = pd.to_numeric(df['known'])
+ df["contents"] = pd.to_numeric(df["contents"])
+ df["known"] = pd.to_numeric(df["known"])
return df
-def sunburst(directories: Dict[PosixPath, Tuple[int, int]],
- root: PosixPath) -> None:
+def sunburst(directories: Dict[PosixPath, Tuple[int, int]], root: PosixPath) -> None:
"""Show the sunburst chart from the directories given in input.
"""
max_depth = compute_max_depth(list(directories.keys()), root)
- metrics_columns = ['contents', 'known']
- levels_columns = ['lev'+str(i) for i in range(max_depth)]
+ metrics_columns = ["contents", "known"]
+ levels_columns = ["lev" + str(i) for i in range(max_depth)]
df_columns = levels_columns + metrics_columns
dirs_df = generate_df_from_dirs(directories, df_columns, root, max_depth)
hierarchical_df = build_hierarchical_df(
- dirs_df, levels_columns, metrics_columns, str(root))
- known_avg = dirs_df['known'].sum() / dirs_df['contents'].sum()
+ dirs_df, levels_columns, metrics_columns, str(root)
+ )
+ known_avg = dirs_df["known"].sum() / dirs_df["contents"].sum()
fig = go.Figure()
- fig.add_trace(go.Sunburst(
- labels=hierarchical_df['id'],
- parents=hierarchical_df['parent'],
- values=hierarchical_df['contents'],
- branchvalues='total',
- marker=dict(
- colors=hierarchical_df['known'],
- colorscale='RdBu',
- cmid=known_avg),
- hovertemplate='''<b>%{label}</b>
+ fig.add_trace(
+ go.Sunburst(
+ labels=hierarchical_df["id"],
+ parents=hierarchical_df["parent"],
+ values=hierarchical_df["contents"],
+ branchvalues="total",
+ marker=dict(
+ colors=hierarchical_df["known"], colorscale="RdBu", cmid=known_avg
+ ),
+ hovertemplate="""<b>%{label}</b>
<br>Files: %{value}
- <br>Known: <b>%{color:.2f}%</b>''',
- name=''
- ))
+ <br>Known: <b>%{color:.2f}%</b>""",
+ name="",
+ )
+ )
- offline.plot(fig, filename='sunburst.html')
+ offline.plot(fig, filename="sunburst.html")
diff --git a/swh/scanner/scanner.py b/swh/scanner/scanner.py
index 81034f4..bbc4e87 100644
--- a/swh/scanner/scanner.py
+++ b/swh/scanner/scanner.py
@@ -1,143 +1,138 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
import itertools
import asyncio
import aiohttp
from typing import List, Dict, Tuple, Iterator
from pathlib import PosixPath
from .exceptions import error_response
from .model import Tree
from swh.model.cli import pid_of_file, pid_of_dir
-from swh.model.identifiers import (
- parse_persistent_identifier,
- DIRECTORY, CONTENT
-)
+from swh.model.identifiers import parse_persistent_identifier, DIRECTORY, CONTENT
async def pids_discovery(
- pids: List[str], session: aiohttp.ClientSession, api_url: str,
- ) -> Dict[str, Dict[str, bool]]:
+ pids: List[str], session: aiohttp.ClientSession, api_url: str,
+) -> Dict[str, Dict[str, bool]]:
"""API Request to get information about the persistent identifiers given in
input.
Args:
pids: a list of persistent identifier
api_url: url for the API request
Returns:
A dictionary with:
key: persistent identifier searched
value:
value['known'] = True if the pid is found
value['known'] = False if the pid is not found
"""
- endpoint = api_url + 'known/'
+ endpoint = api_url + "known/"
chunk_size = 1000
requests = []
def get_chunk(pids):
for i in range(0, len(pids), chunk_size):
- yield pids[i:i + chunk_size]
+ yield pids[i : i + chunk_size]
async def make_request(pids):
async with session.post(endpoint, json=pids) as resp:
if resp.status != 200:
error_response(resp.reason, resp.status, endpoint)
return await resp.json()
if len(pids) > chunk_size:
for pids_chunk in get_chunk(pids):
- requests.append(asyncio.create_task(
- make_request(pids_chunk)))
+ requests.append(asyncio.create_task(make_request(pids_chunk)))
res = await asyncio.gather(*requests)
# concatenate list of dictionaries
return dict(itertools.chain.from_iterable(e.items() for e in res))
else:
return await make_request(pids)
-def get_subpaths(
- path: PosixPath) -> Iterator[Tuple[PosixPath, str]]:
+def get_subpaths(path: PosixPath) -> Iterator[Tuple[PosixPath, str]]:
"""Find the persistent identifier of the directories and files under a
given path.
Args:
path: the root path
Yields:
pairs of: path, the relative persistent identifier
"""
+
def pid_of(path):
if path.is_dir():
return pid_of_dir(bytes(path))
elif path.is_file() or path.is_symlink():
return pid_of_file(bytes(path))
dirpath, dnames, fnames = next(os.walk(path))
for node in itertools.chain(dnames, fnames):
sub_path = PosixPath(dirpath).joinpath(node)
yield (sub_path, pid_of(sub_path))
async def parse_path(
- path: PosixPath, session: aiohttp.ClientSession, api_url: str
- ) -> Iterator[Tuple[str, str, bool]]:
+ path: PosixPath, session: aiohttp.ClientSession, api_url: str
+) -> Iterator[Tuple[str, str, bool]]:
"""Check if the sub paths of the given path are present in the
archive or not.
Args:
path: the source path
api_url: url for the API request
Returns:
a map containing tuples with: a subpath of the given path,
the pid of the subpath and the result of the api call
"""
parsed_paths = dict(get_subpaths(path))
- parsed_pids = await pids_discovery(
- list(parsed_paths.values()), session, api_url)
+ parsed_pids = await pids_discovery(list(parsed_paths.values()), session, api_url)
def unpack(tup):
subpath, pid = tup
- return (subpath, pid, parsed_pids[pid]['known'])
+ return (subpath, pid, parsed_pids[pid]["known"])
return map(unpack, parsed_paths.items())
-async def run(
- root: PosixPath, api_url: str, source_tree: Tree) -> None:
+async def run(root: PosixPath, api_url: str, source_tree: Tree) -> None:
"""Start scanning from the given root.
It fills the source tree with the path discovered.
Args:
root: the root path to scan
api_url: url for the API request
"""
+
async def _scan(root, session, api_url, source_tree):
for path, pid, found in await parse_path(root, session, api_url):
obj_type = parse_persistent_identifier(pid).object_type
if obj_type == CONTENT:
source_tree.addNode(path, pid if found else None)
elif obj_type == DIRECTORY:
if found:
source_tree.addNode(path, pid)
else:
source_tree.addNode(path)
await _scan(path, session, api_url, source_tree)
async with aiohttp.ClientSession() as session:
await _scan(root, session, api_url, source_tree)
diff --git a/swh/scanner/tests/conftest.py b/swh/scanner/tests/conftest.py
index a932de1..fafcc6c 100644
--- a/swh/scanner/tests/conftest.py
+++ b/swh/scanner/tests/conftest.py
@@ -1,136 +1,136 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import pytest
import asyncio
import aiohttp
import os
from pathlib import PosixPath
from aioresponses import aioresponses # type: ignore
from swh.model.cli import pid_of_file, pid_of_dir
from swh.scanner.model import Tree
from .flask_api import create_app
@pytest.fixture
def mock_aioresponse():
with aioresponses() as m:
yield m
@pytest.fixture
def event_loop():
"""Fixture that generate an asyncio event loop."""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
yield loop
loop.close()
@pytest.fixture
async def aiosession():
"""Fixture that generate an aiohttp Client Session."""
session = aiohttp.ClientSession()
yield session
session.detach()
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
def temp_folder(tmp_path_factory):
"""Fixture that generates a temporary folder with the following
structure:
.. code-block:: python
root = {
subdir: {
subsubdir
filesample.txt
filesample2.txt
}
subdir2
subfile.txt
}
"""
root = tmp_path_factory.getbasetemp()
- subdir = tmp_path_factory.mktemp('subdir')
- subsubdir = subdir.joinpath('subsubdir')
+ subdir = tmp_path_factory.mktemp("subdir")
+ subsubdir = subdir.joinpath("subsubdir")
subsubdir.mkdir()
- subdir2 = tmp_path_factory.mktemp('subdir2')
- subfile = root / 'subfile.txt'
+ subdir2 = tmp_path_factory.mktemp("subdir2")
+ subfile = root / "subfile.txt"
subfile.touch()
- filesample = subdir / 'filesample.txt'
+ filesample = subdir / "filesample.txt"
filesample.touch()
- filesample2 = subdir / 'filesample2.txt'
+ filesample2 = subdir / "filesample2.txt"
filesample2.touch()
avail_path = {
subdir: pid_of_dir(bytes(subdir)),
subsubdir: pid_of_dir(bytes(subsubdir)),
subdir2: pid_of_dir(bytes(subdir2)),
subfile: pid_of_file(bytes(subfile)),
filesample: pid_of_file(bytes(filesample)),
- filesample2: pid_of_file(bytes(filesample2))
- }
+ filesample2: pid_of_file(bytes(filesample2)),
+ }
return {
- 'root': root,
- 'paths': avail_path,
- 'filesample': filesample,
- 'filesample2': filesample2,
- 'subsubdir': subsubdir,
- 'subdir': subdir
+ "root": root,
+ "paths": avail_path,
+ "filesample": filesample,
+ "filesample2": filesample2,
+ "subsubdir": subsubdir,
+ "subdir": subdir,
}
-@pytest.fixture(scope='function')
+@pytest.fixture(scope="function")
def example_tree(temp_folder):
"""Fixture that generate a Tree with the root present in the
session fixture "temp_folder".
"""
- example_tree = Tree(temp_folder['root'])
- assert example_tree.path == temp_folder['root']
+ example_tree = Tree(temp_folder["root"])
+ assert example_tree.path == temp_folder["root"]
return example_tree
-@pytest.fixture(scope='function')
+@pytest.fixture(scope="function")
def example_dirs(example_tree, temp_folder):
"""
Fixture that fill the fixture example_tree with the values contained in
the fixture temp_folder and returns the directories information of the
filled example_tree.
"""
- root = temp_folder['root']
- filesample_path = temp_folder['filesample']
- filesample2_path = temp_folder['filesample2']
- subsubdir_path = temp_folder['subsubdir']
+ root = temp_folder["root"]
+ filesample_path = temp_folder["filesample"]
+ filesample2_path = temp_folder["filesample2"]
+ subsubdir_path = temp_folder["subsubdir"]
known_paths = [filesample_path, filesample2_path, subsubdir_path]
- for path, pid in temp_folder['paths'].items():
+ for path, pid in temp_folder["paths"].items():
if path in known_paths:
example_tree.addNode(path, pid)
else:
example_tree.addNode(path)
return example_tree.getDirectoriesInfo(root)
@pytest.fixture
def test_folder():
"""Location of the "data" folder """
tests_path = PosixPath(os.path.abspath(__file__)).parent
- tests_data_folder = tests_path.joinpath('data')
+ tests_data_folder = tests_path.joinpath("data")
assert tests_data_folder.exists()
return tests_data_folder
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
def app():
"""Flask backend API (used by live_server)."""
app = create_app()
return app
diff --git a/swh/scanner/tests/data.py b/swh/scanner/tests/data.py
index 08429c5..4648b3e 100644
--- a/swh/scanner/tests/data.py
+++ b/swh/scanner/tests/data.py
@@ -1,19 +1,17 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
correct_api_response = {
- "swh:1:dir:17d207da3804cc60a77cba58e76c3b2f767cb112":
- {"known": False},
- "swh:1:dir:01fa282bb80be5907505d44b4692d3fa40fad140":
- {"known": True},
- "swh:1:dir:4b825dc642cb6eb9a060e54bf8d69288fbee4904":
- {"known": True}}
+ "swh:1:dir:17d207da3804cc60a77cba58e76c3b2f767cb112": {"known": False},
+ "swh:1:dir:01fa282bb80be5907505d44b4692d3fa40fad140": {"known": True},
+ "swh:1:dir:4b825dc642cb6eb9a060e54bf8d69288fbee4904": {"known": True},
+}
# present pids inside /data/sample-folder
present_pids = [
- "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a", # quotes.md
- "swh:1:cnt:68769579c3eaadbe555379b9c3538e6628bae1eb", # some-binary
- "swh:1:dir:9619a28687b2462efbb5be816bc1185b95753d93" # barfoo2/
- ]
+ "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a", # quotes.md
+ "swh:1:cnt:68769579c3eaadbe555379b9c3538e6628bae1eb", # some-binary
+ "swh:1:dir:9619a28687b2462efbb5be816bc1185b95753d93", # barfoo2/
+]
diff --git a/swh/scanner/tests/flask_api.py b/swh/scanner/tests/flask_api.py
index b24211b..7896d82 100644
--- a/swh/scanner/tests/flask_api.py
+++ b/swh/scanner/tests/flask_api.py
@@ -1,31 +1,32 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from flask import Flask, request
from .data import present_pids
from swh.web.common.exc import LargePayloadExc
def create_app():
app = Flask(__name__)
- @app.route('/known/', methods=['POST'])
+ @app.route("/known/", methods=["POST"])
def known():
pids = request.get_json()
if len(pids) > 900:
- raise LargePayloadExc('The maximum number of PIDs this endpoint '
- 'can receive is 900')
+ raise LargePayloadExc(
+ "The maximum number of PIDs this endpoint " "can receive is 900"
+ )
- res = {pid: {'known': False} for pid in pids}
+ res = {pid: {"known": False} for pid in pids}
for pid in pids:
if pid in present_pids:
- res[pid]['known'] = True
+ res[pid]["known"] = True
return res
return app
diff --git a/swh/scanner/tests/test_model.py b/swh/scanner/tests/test_model.py
index 904356e..5e2757c 100644
--- a/swh/scanner/tests/test_model.py
+++ b/swh/scanner/tests/test_model.py
@@ -1,71 +1,71 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
def test_tree_add_node(example_tree, temp_folder):
- avail_paths = temp_folder['paths'].keys()
+ avail_paths = temp_folder["paths"].keys()
- for path, pid in temp_folder['paths'].items():
+ for path, pid in temp_folder["paths"].items():
example_tree.addNode(path, pid)
for path, node in example_tree.children.items():
assert path in avail_paths
if node.children:
for subpath, subnode in node.children.items():
assert subpath in avail_paths
def test_get_json_tree_all_not_present(example_tree, temp_folder):
- for path, pid in temp_folder['paths'].items():
+ for path, pid in temp_folder["paths"].items():
example_tree.addNode(path)
json_tree = example_tree.getTree()
assert len(json_tree) == 0
def test_get_json_tree_all_present(example_tree, temp_folder):
- for path, pid in temp_folder['paths'].items():
+ for path, pid in temp_folder["paths"].items():
example_tree.addNode(path, pid)
tree_dict = example_tree.getTree()
assert len(tree_dict) == 3
# since subdir have a pid, it can't have a children path
- assert tree_dict['subdir0'] is not dict
+ assert tree_dict["subdir0"] is not dict
def test_get_json_tree_only_one_present(example_tree, temp_folder):
- filesample_path = temp_folder['filesample']
+ filesample_path = temp_folder["filesample"]
- for path, pid in temp_folder['paths'].items():
+ for path, pid in temp_folder["paths"].items():
if path == filesample_path:
example_tree.addNode(path, pid)
else:
example_tree.addNode(path)
tree_dict = example_tree.getTree()
assert len(tree_dict) == 1
- assert tree_dict['subdir0']['filesample.txt']
+ assert tree_dict["subdir0"]["filesample.txt"]
def test_get_directories_info(example_tree, temp_folder):
- root_path = temp_folder['root']
- filesample_path = temp_folder['filesample']
- filesample2_path = temp_folder['filesample2']
- subdir_path = temp_folder['subdir'].relative_to(root_path)
- subsubdir_path = temp_folder['subsubdir'].relative_to(root_path)
+ root_path = temp_folder["root"]
+ filesample_path = temp_folder["filesample"]
+ filesample2_path = temp_folder["filesample2"]
+ subdir_path = temp_folder["subdir"].relative_to(root_path)
+ subsubdir_path = temp_folder["subsubdir"].relative_to(root_path)
- for path, pid in temp_folder['paths'].items():
+ for path, pid in temp_folder["paths"].items():
if path == filesample_path or path == filesample2_path:
example_tree.addNode(path, pid)
else:
example_tree.addNode(path)
directories = example_tree.getDirectoriesInfo(example_tree.path)
assert subsubdir_path not in directories
assert directories[subdir_path] == (2, 2)
diff --git a/swh/scanner/tests/test_plot.py b/swh/scanner/tests/test_plot.py
index a1eb56e..68a926e 100644
--- a/swh/scanner/tests/test_plot.py
+++ b/swh/scanner/tests/test_plot.py
@@ -1,56 +1,57 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.scanner.plot import (
- compute_max_depth, generate_df_from_dirs, build_hierarchical_df
+ compute_max_depth,
+ generate_df_from_dirs,
+ build_hierarchical_df,
)
def test_max_depth(temp_folder, example_dirs):
- root = temp_folder['root']
+ root = temp_folder["root"]
max_depth = compute_max_depth(example_dirs, root)
assert max_depth == 2
def test_generate_df_from_dirs(temp_folder, example_dirs):
- root = temp_folder['root']
+ root = temp_folder["root"]
max_depth = compute_max_depth(example_dirs, root)
- metrics_columns = ['contents', 'known']
- levels_columns = ['lev'+str(i) for i in range(max_depth)]
+ metrics_columns = ["contents", "known"]
+ levels_columns = ["lev" + str(i) for i in range(max_depth)]
df_columns = levels_columns + metrics_columns
- actual_df = generate_df_from_dirs(
- example_dirs, df_columns, root, max_depth)
+ actual_df = generate_df_from_dirs(example_dirs, df_columns, root, max_depth)
# assert root is empty
- assert actual_df['lev0'][0] == ''
- assert actual_df['lev1'][0] == ''
+ assert actual_df["lev0"][0] == ""
+ assert actual_df["lev1"][0] == ""
# assert subdir has correct contents information
- assert actual_df['contents'][1] == 2
- assert actual_df['known'][1] == 2
+ assert actual_df["contents"][1] == 2
+ assert actual_df["known"][1] == 2
# assert subsubdir has correct level information
- assert actual_df['lev0'][2] == 'subdir0'
- assert actual_df['lev1'][2] == 'subdir0/subsubdir'
+ assert actual_df["lev0"][2] == "subdir0"
+ assert actual_df["lev1"][2] == "subdir0/subsubdir"
def test_build_hierarchical_df(temp_folder, example_dirs):
- root = temp_folder['root']
+ root = temp_folder["root"]
max_depth = compute_max_depth(example_dirs, root)
- metrics_columns = ['contents', 'known']
- levels_columns = ['lev'+str(i) for i in range(max_depth)]
+ metrics_columns = ["contents", "known"]
+ levels_columns = ["lev" + str(i) for i in range(max_depth)]
df_columns = levels_columns + metrics_columns
- actual_df = generate_df_from_dirs(
- example_dirs, df_columns, root, max_depth)
+ actual_df = generate_df_from_dirs(example_dirs, df_columns, root, max_depth)
actual_result = build_hierarchical_df(
- actual_df, levels_columns, metrics_columns, root)
+ actual_df, levels_columns, metrics_columns, root
+ )
- assert actual_result['parent'][1] == 'subdir0'
- assert actual_result['contents'][1] == 2
- assert actual_result['id'][5] == root
- assert actual_result['known'][5] == 75
+ assert actual_result["parent"][1] == "subdir0"
+ assert actual_result["contents"][1] == 2
+ assert actual_result["id"][5] == root
+ assert actual_result["known"][5] == 75
diff --git a/swh/scanner/tests/test_scanner.py b/swh/scanner/tests/test_scanner.py
index e6200c9..8404a71 100644
--- a/swh/scanner/tests/test_scanner.py
+++ b/swh/scanner/tests/test_scanner.py
@@ -1,79 +1,82 @@
# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import pytest
import json
from pathlib import PosixPath
from .data import correct_api_response
from swh.scanner.scanner import pids_discovery, get_subpaths, run
from swh.scanner.model import Tree
from swh.scanner.exceptions import APIError
-aio_url = 'http://example.org/api/known/'
+aio_url = "http://example.org/api/known/"
def test_scanner_correct_api_request(mock_aioresponse, event_loop, aiosession):
- mock_aioresponse.post(aio_url, status=200, content_type='application/json',
- body=json.dumps(correct_api_response))
+ mock_aioresponse.post(
+ aio_url,
+ status=200,
+ content_type="application/json",
+ body=json.dumps(correct_api_response),
+ )
actual_result = event_loop.run_until_complete(
- pids_discovery([], aiosession, 'http://example.org/api/'))
+ pids_discovery([], aiosession, "http://example.org/api/")
+ )
assert correct_api_response == actual_result
def test_scanner_raise_apierror(mock_aioresponse, event_loop, aiosession):
- mock_aioresponse.post(aio_url, content_type='application/json',
- status=413)
+ mock_aioresponse.post(aio_url, content_type="application/json", status=413)
with pytest.raises(APIError):
event_loop.run_until_complete(
- pids_discovery([], aiosession, 'http://example.org/api/'))
+ pids_discovery([], aiosession, "http://example.org/api/")
+ )
-def test_scanner_raise_apierror_input_size_limit(
- event_loop, aiosession, live_server):
+def test_scanner_raise_apierror_input_size_limit(event_loop, aiosession, live_server):
- api_url = live_server.url() + '/'
- request = ["swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a"
- for i in range(901)] # /known/ is limited at 900
+ api_url = live_server.url() + "/"
+ request = [
+ "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a" for i in range(901)
+ ] # /known/ is limited at 900
with pytest.raises(APIError):
- event_loop.run_until_complete(
- pids_discovery(request, aiosession, api_url))
+ event_loop.run_until_complete(pids_discovery(request, aiosession, api_url))
def test_scanner_get_subpaths(temp_folder, tmp_path):
- paths = temp_folder['paths'].keys()
- pids = temp_folder['paths'].values()
+ paths = temp_folder["paths"].keys()
+ pids = temp_folder["paths"].values()
for subpath, pid in get_subpaths(tmp_path):
assert subpath in paths
assert pid in pids
@pytest.mark.options(debug=False)
def test_app(app):
assert not app.debug
def test_scanner_result(live_server, event_loop, test_folder):
- api_url = live_server.url() + '/'
+ api_url = live_server.url() + "/"
- result_path = test_folder.joinpath(PosixPath('sample-folder-result.json'))
- with open(result_path, 'r') as json_file:
+ result_path = test_folder.joinpath(PosixPath("sample-folder-result.json"))
+ with open(result_path, "r") as json_file:
expected_result = json.loads(json_file.read())
- sample_folder = test_folder.joinpath(PosixPath('sample-folder'))
+ sample_folder = test_folder.joinpath(PosixPath("sample-folder"))
source_tree = Tree(sample_folder)
- event_loop.run_until_complete(
- run(sample_folder, api_url, source_tree))
+ event_loop.run_until_complete(run(sample_folder, api_url, source_tree))
actual_result = source_tree.getTree()
assert actual_result == expected_result
diff --git a/tox.ini b/tox.ini
index e294617..cbdb899 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,28 +1,35 @@
[tox]
-envlist=flake8,mypy,py3
+envlist=black,flake8,mypy,py3
[testenv]
extras =
testing
deps =
pytest-cov
commands =
pytest --doctest-modules \
{envsitepackagesdir}/swh/scanner \
--cov={envsitepackagesdir}/swh/scanner \
--cov-branch {posargs}
+[testenv:black]
+skip_install = true
+deps =
+ black
+commands =
+ {envpython} -m black --check swh
+
[testenv:flake8]
skip_install = true
deps =
flake8
commands =
{envpython} -m flake8
[testenv:mypy]
extras =
testing
deps =
mypy
commands =
mypy swh