diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,4 +4,7 @@
vcversioner
requests
aiohttp
+plotly
+pandas
+numpy
dulwich
diff --git a/swh/scanner/cli.py b/swh/scanner/cli.py
--- a/swh/scanner/cli.py
+++ b/swh/scanner/cli.py
@@ -35,7 +35,8 @@
metavar='API_URL', show_default=True,
help="url for the api request")
@click.option('-f', '--format',
- type=click.Choice(['text', 'json'], case_sensitive=False),
+ type=click.Choice(['text', 'json', 'sunburst'],
+ case_sensitive=False),
default='text',
help="select the output format")
@click.pass_context
diff --git a/swh/scanner/exceptions.py b/swh/scanner/exceptions.py
--- a/swh/scanner/exceptions.py
+++ b/swh/scanner/exceptions.py
@@ -4,6 +4,10 @@
# See top-level LICENSE file for more information
+class InvalidObjectType(TypeError):
+ pass
+
+
class APIError(Exception):
def __str__(self):
return '"%s"' % self.args
diff --git a/swh/scanner/model.py b/swh/scanner/model.py
--- a/swh/scanner/model.py
+++ b/swh/scanner/model.py
@@ -7,9 +7,12 @@
import sys
import json
from pathlib import PosixPath
-from typing import Any, Dict
+from typing import Any, Dict, Tuple
from enum import Enum
+from .plot import sunburst
+from .exceptions import InvalidObjectType
+
from swh.model.identifiers import (
DIRECTORY, CONTENT
)
@@ -37,7 +40,7 @@
self.children: Dict[PosixPath, Tree] = {}
def addNode(self, path: PosixPath, pid: str = None) -> None:
- """Recursively add a new node path
+ """Recursively add a new path.
"""
relative_path = path.relative_to(self.path)
@@ -53,9 +56,10 @@
self.children[new_path].addNode(path, pid)
def show(self, format) -> None:
- """Print all the tree"""
+ """Show tree in different formats"""
if format == 'json':
print(json.dumps(self.getTree(), indent=4, sort_keys=True))
+
elif format == 'text':
isatty = sys.stdout.isatty()
@@ -63,7 +67,12 @@
else str(self.path))
self.printChildren(isatty)
- def printChildren(self, isatty: bool, inc: int = 0) -> None:
+ elif format == 'sunburst':
+ root = self.path
+ directories = self.getDirectoriesInfo(root)
+ sunburst(directories, root)
+
+ def printChildren(self, isatty: bool, inc: int = 1) -> None:
for path, node in self.children.items():
self.printNode(node, isatty, inc)
if node.children:
@@ -104,3 +113,64 @@
child_tree[rel_path] = next_tree
return child_tree
+
+ def getSubDirsInfo(self, root, directories):
+ """Get information about all directories under a given root.
+ """
+ for path, child_node in self.children.items():
+ if child_node.otype == DIRECTORY:
+ rel_path = path.relative_to(root)
+ contents_info = child_node.count_contents()
+ if not contents_info[0] == 0:
+ directories[rel_path] = contents_info
+ if child_node.has_dirs():
+ child_node.getSubDirsInfo(root, directories)
+
+ return directories
+
+ def getDirectoriesInfo(self, root) -> Dict[PosixPath, Tuple[int, int]]:
+ """Get information about all directories under the current tree.
+
+ Returns:
+ A dictionary with a directory path as key and the relative
+ contents information (the result of count_contents) as values.
+
+ """
+ directories = {root: self.count_contents()}
+ return self.getSubDirsInfo(root, directories)
+
+ def count_contents(self) -> Tuple[int, int]:
+ """Count how many contents are present inside a directory.
+ If a directory has a pid returns as it has all the contents.
+
+ Returns:
+ A tuple with the total number of the contents and the number
+ of contents that have a persistent identifier.
+
+ """
+ contents = 0
+ discovered = 0
+
+ if not self.otype == DIRECTORY:
+ raise InvalidObjectType('Can\'t calculate contents of the '
+ 'object type: %s' % self.otype)
+
+ if self.pid:
+ # to identify a directory with all files/directories present
+ return (1, 1)
+ else:
+ for _, child_node in self.children.items():
+ if child_node.otype == CONTENT:
+ contents += 1
+ if child_node.pid:
+ discovered += 1
+
+ return (contents, discovered)
+
+ def has_dirs(self) -> bool:
+ """Checks if node has directories
+ """
+ for _, child_node in self.children.items():
+ if child_node.otype == DIRECTORY:
+ return True
+ return False
diff --git a/swh/scanner/plot.py b/swh/scanner/plot.py
new file mode 100644
--- /dev/null
+++ b/swh/scanner/plot.py
@@ -0,0 +1,165 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from typing import Iterable, List, Dict, Tuple
+from pathlib import PosixPath
+
+import plotly.graph_objects as go # type: ignore
+import pandas as pd # type: ignore
+import numpy as np # type: ignore
+
+
+"""
+The purpose of this module is to display and to interact with the result of the
+scanner contained in the model.
+
+The `sunburst` function generates a navigable sunburst chart from the
+directories information retrieved from the model. The chart displays for
+each directory the total number of files and the percentage of file known.
+
+The size of the directory is defined by the total number of contents whereas
+the color gradient is generated relying on the percentage of contents known.
+"""
+
+
+def build_hierarchical_df(
+ df: pd.DataFrame, levels: List[str],
+ metrics_columns: List[str]) -> pd.DataFrame:
+ """
+ Build a hierarchy of levels for Sunburst or Treemap charts.
+
+ For each directory level store the new dataframe will have:
+ id: the directory name
+ parent: the parent directory of id
+ contents: the number of contents inside the directory id
+ known: the number of contents known of the directory id
+
+ """
+ complete_df = pd.DataFrame(columns=['id', 'parent', 'contents', 'known'])
+ # revert the level order to start from the deepest
+ levels = [level for level in reversed(levels)]
+ contents_col = metrics_columns[0]
+ known_col = metrics_columns[1]
+
+ for i, level in enumerate(levels):
+ print(i, level)
+ df_tree = pd.DataFrame(columns=['id', 'parent', 'contents', 'known'])
+ dfg = df.groupby(levels[i:]).sum()
+ dfg = dfg.reset_index()
+ df_tree['id'] = dfg[level].copy()
+ if i < len(levels) - 1:
+ df_tree['parent'] = dfg[levels[i+1]].copy()
+ else:
+ df_tree['parent'] = 'total'
+ df_tree['contents'] = dfg[contents_col]
+ df_tree['known'] = dfg[known_col] / dfg[contents_col] * 100
+ complete_df = complete_df.append(df_tree, ignore_index=True)
+
+ tot_avg = df[known_col].sum() / df[contents_col].sum() * 100
+ total = pd.Series(dict(id='total', parent='',
+ contents=df[contents_col].sum(),
+ known=tot_avg))
+
+ complete_df = complete_df.append(total, ignore_index=True)
+
+ return complete_df
+
+
+def compute_max_depth(dirs_path, root: PosixPath) -> int:
+ """Compute the maximum depth level of the given directory paths.
+
+ Example: for `var/log/kernel/` the depth level is 3
+
+ """
+ max_depth = 0
+ for dir_path in dirs_path:
+ if dir_path == root:
+ continue
+
+ dir_depth = len(dir_path.parts)
+ if dir_depth > max_depth:
+ max_depth = dir_depth
+
+ return max_depth
+
+
+def generate_df(dirs, columns: List[str], root: PosixPath,
+ max_depth: int) -> pd.DataFrame:
+ """Generate a dataframe from the directories given in input.
+
+ Example:
+ given the following directories as input
+
+ .. code-block:: python
+
+ dirs = {
+ '/var/log/': (23, 2),
+ '/var/log/kernel': (5, 0),
+ '/var/log/telnet': (10, 3)
+ }
+
+ The generated dataframe will be:
+
+ .. code-block:: none
+
+ lev0 lev1 lev2 contents known
+ 'var' 'log' '' 23 2
+ 'var' 'log' 'kernel' 5 0
+ 'var' 'log' 'telnet' 10 3
+
+ """
+ def get_dirs_array(dir_path: PosixPath) -> Iterable[List[str]]:
+ for dir_path, contents_info in dirs.items():
+ empty_lvl = max_depth - len(dir_path.parts)
+
+ if dir_path == root:
+ # ignore the root but store contents information
+ yield ['']*(max_depth) + list(contents_info)
+ else:
+ path_array = [part for part in dir_path.parts]
+ yield path_array + ['']*empty_lvl + list(contents_info)
+
+ df = pd.DataFrame(np.array(
+ [dir_array for dir_array in get_dirs_array(dirs)]), columns=columns)
+
+ df['contents'] = pd.to_numeric(df['contents'])
+ df['known'] = pd.to_numeric(df['known'])
+
+ return df
+
+
+def sunburst(directories: Dict[PosixPath, Tuple[int, int]],
+ root: PosixPath) -> None:
+ """Show the sunburst chart from the directories given in input.
+
+ """
+ max_depth = compute_max_depth(directories.keys(), root)
+ metrics_columns = ['contents', 'known']
+ levels_columns = ['lev'+str(i) for i in range(max_depth)]
+
+ df_columns = levels_columns + metrics_columns
+ df = generate_df(directories, df_columns, root, max_depth)
+
+ hierarchical_df = build_hierarchical_df(
+ df, levels_columns, metrics_columns)
+ known_avg = df['known'].sum() / df['contents'].sum()
+
+ fig = go.Figure()
+ fig.add_trace(go.Sunburst(
+ labels=hierarchical_df['id'],
+ parents=hierarchical_df['parent'],
+ values=hierarchical_df['contents'],
+ branchvalues='total',
+ marker=dict(
+ colors=hierarchical_df['known'],
+ colorscale='RdBu',
+ cmid=known_avg),
+ hovertemplate='''%{label}
+
Files: %{value}
+
Known: %{color:.2f}%''',
+ name=''
+ ))
+
+ fig.show()
diff --git a/swh/scanner/tests/conftest.py b/swh/scanner/tests/conftest.py
--- a/swh/scanner/tests/conftest.py
+++ b/swh/scanner/tests/conftest.py
@@ -11,6 +11,7 @@
from aioresponses import aioresponses # type: ignore
from swh.model.cli import pid_of_file, pid_of_dir
+from swh.scanner.model import Tree
from .flask_api import create_app
@@ -46,7 +47,9 @@
root = {
subdir: {
+ subsubdir
filesample.txt
+ filesample2.txt
}
subdir2
subfile.txt
@@ -54,31 +57,44 @@
"""
root = tmp_path_factory.getbasetemp()
subdir = tmp_path_factory.mktemp('subdir')
+ subsubdir = subdir.joinpath('subsubdir')
+ subsubdir.mkdir()
subdir2 = tmp_path_factory.mktemp('subdir2')
subfile = root / 'subfile.txt'
subfile.touch()
filesample = subdir / 'filesample.txt'
filesample.touch()
+ filesample2 = subdir / 'filesample2.txt'
+ filesample2.touch()
avail_path = {
subdir: pid_of_dir(bytes(subdir)),
+ subsubdir: pid_of_dir(bytes(subsubdir)),
subdir2: pid_of_dir(bytes(subdir2)),
subfile: pid_of_file(bytes(subfile)),
- filesample: pid_of_file(bytes(filesample))
+ filesample: pid_of_file(bytes(filesample)),
+ filesample2: pid_of_file(bytes(filesample2))
}
return {
'root': root,
'paths': avail_path,
- 'filesample': filesample
+ 'filesample': filesample,
+ 'filesample2': filesample2,
+ 'subsubdir': subsubdir,
+ 'subdir': subdir
}
-@pytest.fixture(scope='session')
-def app():
- """Flask backend API (used by live_server)."""
- app = create_app()
- return app
+@pytest.fixture(scope='function')
+def example_tree(temp_folder):
+ """Fixture that generate a Tree with the root present in the
+ session fixture "temp_folder".
+ """
+ example_tree = Tree(temp_folder['root'])
+ assert example_tree.path == temp_folder['root']
+
+ return example_tree
@pytest.fixture
@@ -88,3 +104,10 @@
tests_data_folder = tests_path.joinpath('data')
assert tests_data_folder.exists()
return tests_data_folder
+
+
+@pytest.fixture(scope='session')
+def app():
+ """Flask backend API (used by live_server)."""
+ app = create_app()
+ return app
diff --git a/swh/scanner/tests/test_model.py b/swh/scanner/tests/test_model.py
--- a/swh/scanner/tests/test_model.py
+++ b/swh/scanner/tests/test_model.py
@@ -3,21 +3,6 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-import pytest
-
-from swh.scanner.model import Tree
-
-
-@pytest.fixture(scope='function')
-def example_tree(temp_folder):
- """Fixture that generate a Tree with the root present in the
- session fixture "temp_folder".
- """
- example_tree = Tree(temp_folder['root'])
- assert example_tree.path == temp_folder['root']
-
- return example_tree
-
def test_tree_add_node(example_tree, temp_folder):
avail_paths = temp_folder['paths'].keys()
@@ -65,3 +50,22 @@
assert len(tree_dict) == 1
assert tree_dict['subdir0']['filesample.txt']
+
+
+def test_get_directories_info(example_tree, temp_folder):
+ root_path = temp_folder['root']
+ filesample_path = temp_folder['filesample']
+ filesample2_path = temp_folder['filesample2']
+ subdir_path = temp_folder['subdir'].relative_to(root_path)
+ subsubdir_path = temp_folder['subsubdir'].relative_to(root_path)
+
+ for path, pid in temp_folder['paths'].items():
+ if path == filesample_path or path == filesample2_path:
+ example_tree.addNode(path, pid)
+ else:
+ example_tree.addNode(path)
+
+ directories = example_tree.getDirectoriesInfo(example_tree.path)
+
+ assert subsubdir_path not in directories
+ assert directories[subdir_path] == (2, 2)
diff --git a/swh/scanner/tests/test_plot.py b/swh/scanner/tests/test_plot.py
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/test_plot.py
@@ -0,0 +1,41 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.scanner.plot import generate_df, compute_max_depth
+
+
+def test_generate_df(example_tree, temp_folder):
+ root = temp_folder['root']
+ filesample_path = temp_folder['filesample']
+ filesample2_path = temp_folder['filesample2']
+ subsubdir_path = temp_folder['subsubdir']
+ known_paths = [filesample_path, filesample2_path, subsubdir_path]
+
+ for path, pid in temp_folder['paths'].items():
+ if path in known_paths:
+ example_tree.addNode(path, pid)
+ else:
+ example_tree.addNode(path)
+
+ dirs = example_tree.getDirectoriesInfo(root)
+ max_depth = compute_max_depth(dirs, root)
+ assert max_depth == 2
+
+ metrics_columns = ['contents', 'known']
+ levels_columns = ['lev'+str(i) for i in range(max_depth)]
+ df_columns = levels_columns + metrics_columns
+ actual_df = generate_df(dirs, df_columns, root, max_depth)
+
+ # assert root is empty
+ assert actual_df['lev0'][0] == ''
+ assert actual_df['lev1'][0] == ''
+
+ # assert subdir has correct contents information
+ assert actual_df['contents'][1] == 2
+ assert actual_df['known'][1] == 2
+
+ # assert subsubdir has correct level information
+ assert actual_df['lev0'][2] == 'subdir0'
+ assert actual_df['lev1'][2] == 'subsubdir'