diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,7 @@ vcversioner requests aiohttp +plotly +pandas +numpy dulwich diff --git a/swh/scanner/cli.py b/swh/scanner/cli.py --- a/swh/scanner/cli.py +++ b/swh/scanner/cli.py @@ -35,7 +35,8 @@ metavar='API_URL', show_default=True, help="url for the api request") @click.option('-f', '--format', - type=click.Choice(['text', 'json'], case_sensitive=False), + type=click.Choice(['text', 'json', 'sunburst'], + case_sensitive=False), default='text', help="select the output format") @click.pass_context diff --git a/swh/scanner/exceptions.py b/swh/scanner/exceptions.py --- a/swh/scanner/exceptions.py +++ b/swh/scanner/exceptions.py @@ -4,6 +4,10 @@ # See top-level LICENSE file for more information +class InvalidObjectType(TypeError): + pass + + class APIError(Exception): def __str__(self): return '"%s"' % self.args diff --git a/swh/scanner/model.py b/swh/scanner/model.py --- a/swh/scanner/model.py +++ b/swh/scanner/model.py @@ -7,9 +7,12 @@ import sys import json from pathlib import PosixPath -from typing import Any, Dict +from typing import Any, Dict, Tuple from enum import Enum +from .plot import sunburst +from .exceptions import InvalidObjectType + from swh.model.identifiers import ( DIRECTORY, CONTENT ) @@ -37,7 +40,7 @@ self.children: Dict[PosixPath, Tree] = {} def addNode(self, path: PosixPath, pid: str = None) -> None: - """Recursively add a new node path + """Recursively add a new path. """ relative_path = path.relative_to(self.path) @@ -53,9 +56,10 @@ self.children[new_path].addNode(path, pid) def show(self, format) -> None: - """Print all the tree""" + """Show tree in different formats""" if format == 'json': print(json.dumps(self.getTree(), indent=4, sort_keys=True)) + elif format == 'text': isatty = sys.stdout.isatty() @@ -63,7 +67,12 @@ else str(self.path)) self.printChildren(isatty) - def printChildren(self, isatty: bool, inc: int = 0) -> None: + elif format == 'sunburst': + root = self.path + directories = self.getDirectoriesInfo(root) + sunburst(directories, root) + + def printChildren(self, isatty: bool, inc: int = 1) -> None: for path, node in self.children.items(): self.printNode(node, isatty, inc) if node.children: @@ -104,3 +113,64 @@ child_tree[rel_path] = next_tree return child_tree + + def getSubDirsInfo(self, root, directories): + """Get information about all directories under a given root. + """ + for path, child_node in self.children.items(): + if child_node.otype == DIRECTORY: + rel_path = path.relative_to(root) + contents_info = child_node.count_contents() + if not contents_info[0] == 0: + directories[rel_path] = contents_info + if child_node.has_dirs(): + child_node.getSubDirsInfo(root, directories) + + return directories + + def getDirectoriesInfo(self, root) -> Dict[PosixPath, Tuple[int, int]]: + """Get information about all directories under the current tree. + + Returns: + A dictionary with a directory path as key and the relative + contents information (the result of count_contents) as values. + + """ + directories = {root: self.count_contents()} + return self.getSubDirsInfo(root, directories) + + def count_contents(self) -> Tuple[int, int]: + """Count how many contents are present inside a directory. + If a directory has a pid returns as it has all the contents. + + Returns: + A tuple with the total number of the contents and the number + of contents that have a persistent identifier. + + """ + contents = 0 + discovered = 0 + + if not self.otype == DIRECTORY: + raise InvalidObjectType('Can\'t calculate contents of the ' + 'object type: %s' % self.otype) + + if self.pid: + # to identify a directory with all files/directories present + return (1, 1) + else: + for _, child_node in self.children.items(): + if child_node.otype == CONTENT: + contents += 1 + if child_node.pid: + discovered += 1 + + return (contents, discovered) + + def has_dirs(self) -> bool: + """Checks if node has directories + """ + for _, child_node in self.children.items(): + if child_node.otype == DIRECTORY: + return True + return False diff --git a/swh/scanner/plot.py b/swh/scanner/plot.py new file mode 100644 --- /dev/null +++ b/swh/scanner/plot.py @@ -0,0 +1,165 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from typing import Iterable, List, Dict, Tuple +from pathlib import PosixPath + +import plotly.graph_objects as go # type: ignore +import pandas as pd # type: ignore +import numpy as np # type: ignore + + +""" +The purpose of this module is to display and to interact with the result of the +scanner contained in the model. + +The `sunburst` function generates a navigable sunburst chart from the +directories information retrieved from the model. The chart displays for +each directory the total number of files and the percentage of file known. + +The size of the directory is defined by the total number of contents whereas +the color gradient is generated relying on the percentage of contents known. +""" + + +def build_hierarchical_df( + df: pd.DataFrame, levels: List[str], + metrics_columns: List[str]) -> pd.DataFrame: + """ + Build a hierarchy of levels for Sunburst or Treemap charts. + + For each directory level store the new dataframe will have: + id: the directory name + parent: the parent directory of id + contents: the number of contents inside the directory id + known: the number of contents known of the directory id + + """ + complete_df = pd.DataFrame(columns=['id', 'parent', 'contents', 'known']) + # revert the level order to start from the deepest + levels = [level for level in reversed(levels)] + contents_col = metrics_columns[0] + known_col = metrics_columns[1] + + for i, level in enumerate(levels): + print(i, level) + df_tree = pd.DataFrame(columns=['id', 'parent', 'contents', 'known']) + dfg = df.groupby(levels[i:]).sum() + dfg = dfg.reset_index() + df_tree['id'] = dfg[level].copy() + if i < len(levels) - 1: + df_tree['parent'] = dfg[levels[i+1]].copy() + else: + df_tree['parent'] = 'total' + df_tree['contents'] = dfg[contents_col] + df_tree['known'] = dfg[known_col] / dfg[contents_col] * 100 + complete_df = complete_df.append(df_tree, ignore_index=True) + + tot_avg = df[known_col].sum() / df[contents_col].sum() * 100 + total = pd.Series(dict(id='total', parent='', + contents=df[contents_col].sum(), + known=tot_avg)) + + complete_df = complete_df.append(total, ignore_index=True) + + return complete_df + + +def compute_max_depth(dirs_path, root: PosixPath) -> int: + """Compute the maximum depth level of the given directory paths. + + Example: for `var/log/kernel/` the depth level is 3 + + """ + max_depth = 0 + for dir_path in dirs_path: + if dir_path == root: + continue + + dir_depth = len(dir_path.parts) + if dir_depth > max_depth: + max_depth = dir_depth + + return max_depth + + +def generate_df(dirs, columns: List[str], root: PosixPath, + max_depth: int) -> pd.DataFrame: + """Generate a dataframe from the directories given in input. + + Example: + given the following directories as input + + .. code-block:: python + + dirs = { + '/var/log/': (23, 2), + '/var/log/kernel': (5, 0), + '/var/log/telnet': (10, 3) + } + + The generated dataframe will be: + + .. code-block:: none + + lev0 lev1 lev2 contents known + 'var' 'log' '' 23 2 + 'var' 'log' 'kernel' 5 0 + 'var' 'log' 'telnet' 10 3 + + """ + def get_dirs_array(dir_path: PosixPath) -> Iterable[List[str]]: + for dir_path, contents_info in dirs.items(): + empty_lvl = max_depth - len(dir_path.parts) + + if dir_path == root: + # ignore the root but store contents information + yield ['']*(max_depth) + list(contents_info) + else: + path_array = [part for part in dir_path.parts] + yield path_array + ['']*empty_lvl + list(contents_info) + + df = pd.DataFrame(np.array( + [dir_array for dir_array in get_dirs_array(dirs)]), columns=columns) + + df['contents'] = pd.to_numeric(df['contents']) + df['known'] = pd.to_numeric(df['known']) + + return df + + +def sunburst(directories: Dict[PosixPath, Tuple[int, int]], + root: PosixPath) -> None: + """Show the sunburst chart from the directories given in input. + + """ + max_depth = compute_max_depth(directories.keys(), root) + metrics_columns = ['contents', 'known'] + levels_columns = ['lev'+str(i) for i in range(max_depth)] + + df_columns = levels_columns + metrics_columns + df = generate_df(directories, df_columns, root, max_depth) + + hierarchical_df = build_hierarchical_df( + df, levels_columns, metrics_columns) + known_avg = df['known'].sum() / df['contents'].sum() + + fig = go.Figure() + fig.add_trace(go.Sunburst( + labels=hierarchical_df['id'], + parents=hierarchical_df['parent'], + values=hierarchical_df['contents'], + branchvalues='total', + marker=dict( + colors=hierarchical_df['known'], + colorscale='RdBu', + cmid=known_avg), + hovertemplate='''%{label} +
Files: %{value} +
Known: %{color:.2f}%''', + name='' + )) + + fig.show() diff --git a/swh/scanner/tests/conftest.py b/swh/scanner/tests/conftest.py --- a/swh/scanner/tests/conftest.py +++ b/swh/scanner/tests/conftest.py @@ -11,6 +11,7 @@ from aioresponses import aioresponses # type: ignore from swh.model.cli import pid_of_file, pid_of_dir +from swh.scanner.model import Tree from .flask_api import create_app @@ -46,7 +47,9 @@ root = { subdir: { + subsubdir filesample.txt + filesample2.txt } subdir2 subfile.txt @@ -54,31 +57,44 @@ """ root = tmp_path_factory.getbasetemp() subdir = tmp_path_factory.mktemp('subdir') + subsubdir = subdir.joinpath('subsubdir') + subsubdir.mkdir() subdir2 = tmp_path_factory.mktemp('subdir2') subfile = root / 'subfile.txt' subfile.touch() filesample = subdir / 'filesample.txt' filesample.touch() + filesample2 = subdir / 'filesample2.txt' + filesample2.touch() avail_path = { subdir: pid_of_dir(bytes(subdir)), + subsubdir: pid_of_dir(bytes(subsubdir)), subdir2: pid_of_dir(bytes(subdir2)), subfile: pid_of_file(bytes(subfile)), - filesample: pid_of_file(bytes(filesample)) + filesample: pid_of_file(bytes(filesample)), + filesample2: pid_of_file(bytes(filesample2)) } return { 'root': root, 'paths': avail_path, - 'filesample': filesample + 'filesample': filesample, + 'filesample2': filesample2, + 'subsubdir': subsubdir, + 'subdir': subdir } -@pytest.fixture(scope='session') -def app(): - """Flask backend API (used by live_server).""" - app = create_app() - return app +@pytest.fixture(scope='function') +def example_tree(temp_folder): + """Fixture that generate a Tree with the root present in the + session fixture "temp_folder". + """ + example_tree = Tree(temp_folder['root']) + assert example_tree.path == temp_folder['root'] + + return example_tree @pytest.fixture @@ -88,3 +104,10 @@ tests_data_folder = tests_path.joinpath('data') assert tests_data_folder.exists() return tests_data_folder + + +@pytest.fixture(scope='session') +def app(): + """Flask backend API (used by live_server).""" + app = create_app() + return app diff --git a/swh/scanner/tests/test_model.py b/swh/scanner/tests/test_model.py --- a/swh/scanner/tests/test_model.py +++ b/swh/scanner/tests/test_model.py @@ -3,21 +3,6 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pytest - -from swh.scanner.model import Tree - - -@pytest.fixture(scope='function') -def example_tree(temp_folder): - """Fixture that generate a Tree with the root present in the - session fixture "temp_folder". - """ - example_tree = Tree(temp_folder['root']) - assert example_tree.path == temp_folder['root'] - - return example_tree - def test_tree_add_node(example_tree, temp_folder): avail_paths = temp_folder['paths'].keys() @@ -65,3 +50,22 @@ assert len(tree_dict) == 1 assert tree_dict['subdir0']['filesample.txt'] + + +def test_get_directories_info(example_tree, temp_folder): + root_path = temp_folder['root'] + filesample_path = temp_folder['filesample'] + filesample2_path = temp_folder['filesample2'] + subdir_path = temp_folder['subdir'].relative_to(root_path) + subsubdir_path = temp_folder['subsubdir'].relative_to(root_path) + + for path, pid in temp_folder['paths'].items(): + if path == filesample_path or path == filesample2_path: + example_tree.addNode(path, pid) + else: + example_tree.addNode(path) + + directories = example_tree.getDirectoriesInfo(example_tree.path) + + assert subsubdir_path not in directories + assert directories[subdir_path] == (2, 2) diff --git a/swh/scanner/tests/test_plot.py b/swh/scanner/tests/test_plot.py new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/test_plot.py @@ -0,0 +1,41 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.scanner.plot import generate_df, compute_max_depth + + +def test_generate_df(example_tree, temp_folder): + root = temp_folder['root'] + filesample_path = temp_folder['filesample'] + filesample2_path = temp_folder['filesample2'] + subsubdir_path = temp_folder['subsubdir'] + known_paths = [filesample_path, filesample2_path, subsubdir_path] + + for path, pid in temp_folder['paths'].items(): + if path in known_paths: + example_tree.addNode(path, pid) + else: + example_tree.addNode(path) + + dirs = example_tree.getDirectoriesInfo(root) + max_depth = compute_max_depth(dirs, root) + assert max_depth == 2 + + metrics_columns = ['contents', 'known'] + levels_columns = ['lev'+str(i) for i in range(max_depth)] + df_columns = levels_columns + metrics_columns + actual_df = generate_df(dirs, df_columns, root, max_depth) + + # assert root is empty + assert actual_df['lev0'][0] == '' + assert actual_df['lev1'][0] == '' + + # assert subdir has correct contents information + assert actual_df['contents'][1] == 2 + assert actual_df['known'][1] == 2 + + # assert subsubdir has correct level information + assert actual_df['lev0'][2] == 'subdir0' + assert actual_df['lev1'][2] == 'subsubdir'