Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
# Copyright (C) 2015-2019 The Software Heritage developers | # Copyright (C) 2015-2019 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from collections import defaultdict | |||||
import copy | import copy | ||||
from concurrent.futures import ThreadPoolExecutor | |||||
from contextlib import contextmanager | |||||
import datetime | import datetime | ||||
import itertools | import itertools | ||||
import json | import json | ||||
from collections import defaultdict | |||||
from concurrent.futures import ThreadPoolExecutor | |||||
from contextlib import contextmanager | |||||
from typing import Any, Dict, Mapping | |||||
import dateutil.parser | import dateutil.parser | ||||
import psycopg2 | import psycopg2 | ||||
import psycopg2.pool | import psycopg2.pool | ||||
from . import converters | from . import converters | ||||
from .common import db_transaction_generator, db_transaction | from .common import db_transaction_generator, db_transaction | ||||
from .db import Db | from .db import Db | ||||
from .exc import StorageDBError | from .exc import StorageDBError | ||||
▲ Show 20 Lines • Show All 1,479 Lines • ▼ Show 20 Lines | def origin_get_by_sha1(self, sha1s, db=None, cur=None): | ||||
""" | """ | ||||
for line in db.origin_get_by_sha1(sha1s, cur): | for line in db.origin_get_by_sha1(sha1s, cur): | ||||
if line[0] is not None: | if line[0] is not None: | ||||
yield dict(zip(db.origin_cols, line)) | yield dict(zip(db.origin_cols, line)) | ||||
else: | else: | ||||
yield None | yield None | ||||
@db_transaction() | |||||
def origin_visit_get_random( | |||||
self, type, db=None, cur=None) -> Mapping[str, Any]: | |||||
"""Randomly select one origin from the archive | |||||
Returns: | |||||
origin dict selected randomly on the dataset if found | |||||
""" | |||||
data: Dict[str, Any] = {} | |||||
result = db.origin_visit_get_random(type, cur) | |||||
if result: | |||||
data = dict(zip(db.origin_visit_get_cols, result)) | |||||
return data | |||||
@db_transaction_generator() | @db_transaction_generator() | ||||
def origin_get_range(self, origin_from=1, origin_count=100, | def origin_get_range(self, origin_from=1, origin_count=100, | ||||
db=None, cur=None): | db=None, cur=None): | ||||
"""Retrieve ``origin_count`` origins whose ids are greater | """Retrieve ``origin_count`` origins whose ids are greater | ||||
or equal than ``origin_from``. | or equal than ``origin_from``. | ||||
Origins are sorted by id before retrieving them. | Origins are sorted by id before retrieving them. | ||||
▲ Show 20 Lines • Show All 328 Lines • Show Last 20 Lines |