Changeset View
Changeset View
Standalone View
Standalone View
swh/scanner/scanner.py
# Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import os | import os | ||||
import itertools | import itertools | ||||
import asyncio | import asyncio | ||||
import aiohttp | import aiohttp | ||||
from typing import List, Dict, Tuple, Iterator | from typing import List, Dict, Tuple, Iterator | ||||
from pathlib import PosixPath | from pathlib import PosixPath | ||||
from .exceptions import APIError | from .exceptions import error_response | ||||
from .model import Tree | from .model import Tree | ||||
from swh.model.cli import pid_of_file, pid_of_dir | from swh.model.cli import pid_of_file, pid_of_dir | ||||
from swh.model.identifiers import ( | from swh.model.identifiers import ( | ||||
parse_persistent_identifier, | parse_persistent_identifier, | ||||
DIRECTORY, CONTENT | DIRECTORY, CONTENT | ||||
) | ) | ||||
Show All 22 Lines | async def pids_discovery( | ||||
def get_chunk(pids): | def get_chunk(pids): | ||||
for i in range(0, len(pids), chunk_size): | for i in range(0, len(pids), chunk_size): | ||||
yield pids[i:i + chunk_size] | yield pids[i:i + chunk_size] | ||||
async def make_request(pids): | async def make_request(pids): | ||||
async with session.post(endpoint, json=pids) as resp: | async with session.post(endpoint, json=pids) as resp: | ||||
if resp.status != 200: | if resp.status != 200: | ||||
error_message = '%s with given values %s' % ( | error_response(resp.status, endpoint) | ||||
resp.text, str(pids)) | |||||
raise APIError(error_message) | |||||
return await resp.json() | return await resp.json() | ||||
if len(pids) > chunk_size: | if len(pids) > chunk_size: | ||||
for pids_chunk in get_chunk(pids): | for pids_chunk in get_chunk(pids): | ||||
requests.append(asyncio.create_task( | requests.append(asyncio.create_task( | ||||
make_request(pids_chunk))) | make_request(pids_chunk))) | ||||
▲ Show 20 Lines • Show All 83 Lines • Show Last 20 Lines |