diff --git a/swh/indexer/__init__.py b/swh/indexer/__init__.py --- a/swh/indexer/__init__.py +++ b/swh/indexer/__init__.py @@ -1,4 +1,11 @@ -# Copyright (C) 2016-2018 The Software Heritage developers +# Copyright (C) 2016-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information + + +# implemented as a function to help lazy loading +def get_datastore(*args, **kw): + from .indexer import get_indexer_storage + + return get_indexer_storage(*args, **kw) diff --git a/swh/indexer/sql/30-schema.sql b/swh/indexer/sql/30-schema.sql --- a/swh/indexer/sql/30-schema.sql +++ b/swh/indexer/sql/30-schema.sql @@ -2,19 +2,6 @@ --- Software Heritage Indexers Data Model --- --- drop schema if exists swh cascade; --- create schema swh; --- set search_path to swh; - -create table dbversion -( - version int primary key, - release timestamptz, - description text -); - -insert into dbversion(version, release, description) - values(133, now(), 'Work In Progress'); -- Computing metadata on sha1's contents -- a SHA1 checksum (not necessarily originating from Git) diff --git a/sql/upgrades/115.sql b/swh/indexer/sql/upgrades/115.sql rename from sql/upgrades/115.sql rename to swh/indexer/sql/upgrades/115.sql diff --git a/sql/upgrades/116.sql b/swh/indexer/sql/upgrades/116.sql rename from sql/upgrades/116.sql rename to swh/indexer/sql/upgrades/116.sql diff --git a/sql/upgrades/117.sql b/swh/indexer/sql/upgrades/117.sql rename from sql/upgrades/117.sql rename to swh/indexer/sql/upgrades/117.sql diff --git a/sql/upgrades/118.sql b/swh/indexer/sql/upgrades/118.sql rename from sql/upgrades/118.sql rename to swh/indexer/sql/upgrades/118.sql diff --git a/sql/upgrades/119.sql b/swh/indexer/sql/upgrades/119.sql rename from sql/upgrades/119.sql rename to swh/indexer/sql/upgrades/119.sql diff --git a/sql/upgrades/120.sql b/swh/indexer/sql/upgrades/120.sql rename from sql/upgrades/120.sql rename to swh/indexer/sql/upgrades/120.sql diff --git a/sql/upgrades/121.sql b/swh/indexer/sql/upgrades/121.sql rename from sql/upgrades/121.sql rename to swh/indexer/sql/upgrades/121.sql diff --git a/sql/upgrades/122.sql b/swh/indexer/sql/upgrades/122.sql rename from sql/upgrades/122.sql rename to swh/indexer/sql/upgrades/122.sql diff --git a/sql/upgrades/123.sql b/swh/indexer/sql/upgrades/123.sql rename from sql/upgrades/123.sql rename to swh/indexer/sql/upgrades/123.sql diff --git a/sql/upgrades/124.sql b/swh/indexer/sql/upgrades/124.sql rename from sql/upgrades/124.sql rename to swh/indexer/sql/upgrades/124.sql diff --git a/sql/upgrades/125.sql b/swh/indexer/sql/upgrades/125.sql rename from sql/upgrades/125.sql rename to swh/indexer/sql/upgrades/125.sql diff --git a/sql/upgrades/126.sql b/swh/indexer/sql/upgrades/126.sql rename from sql/upgrades/126.sql rename to swh/indexer/sql/upgrades/126.sql diff --git a/sql/upgrades/127.sql b/swh/indexer/sql/upgrades/127.sql rename from sql/upgrades/127.sql rename to swh/indexer/sql/upgrades/127.sql diff --git a/sql/upgrades/128.sql b/swh/indexer/sql/upgrades/128.sql rename from sql/upgrades/128.sql rename to swh/indexer/sql/upgrades/128.sql diff --git a/sql/upgrades/129.sql b/swh/indexer/sql/upgrades/129.sql rename from sql/upgrades/129.sql rename to swh/indexer/sql/upgrades/129.sql diff --git a/sql/upgrades/130.sql b/swh/indexer/sql/upgrades/130.sql rename from sql/upgrades/130.sql rename to swh/indexer/sql/upgrades/130.sql diff --git a/sql/upgrades/131.sql b/swh/indexer/sql/upgrades/131.sql rename from sql/upgrades/131.sql rename to swh/indexer/sql/upgrades/131.sql diff --git a/sql/upgrades/132.sql b/swh/indexer/sql/upgrades/132.sql rename from sql/upgrades/132.sql rename to swh/indexer/sql/upgrades/132.sql diff --git a/sql/upgrades/133.sql b/swh/indexer/sql/upgrades/133.sql rename from sql/upgrades/133.sql rename to swh/indexer/sql/upgrades/133.sql diff --git a/swh/indexer/storage/__init__.py b/swh/indexer/storage/__init__.py --- a/swh/indexer/storage/__init__.py +++ b/swh/indexer/storage/__init__.py @@ -42,9 +42,11 @@ SERVER_IMPLEMENTATIONS: Dict[str, str] = { - "local": ".IndexerStorage", + "postgresql": ".IndexerStorage", "remote": ".api.client.RemoteStorage", "memory": ".in_memory.IndexerStorage", + # deprecated + "local": ".IndexerStorage", } @@ -152,6 +154,10 @@ if db is not self._db: db.put_conn() + @db_transaction() + def get_current_version(self, *, db=None, cur=None): + return db.current_version + @timed @db_transaction() def check_config(self, *, check_write, db=None, cur=None): diff --git a/swh/indexer/storage/db.py b/swh/indexer/storage/db.py --- a/swh/indexer/storage/db.py +++ b/swh/indexer/storage/db.py @@ -18,6 +18,7 @@ """ content_mimetype_hash_keys = ["id", "indexer_configuration_id"] + current_version = 133 def _missing_from_list( self, table: str, data: Iterable[Dict], hash_keys: List[str], cur=None diff --git a/swh/indexer/tests/conftest.py b/swh/indexer/tests/conftest.py --- a/swh/indexer/tests/conftest.py +++ b/swh/indexer/tests/conftest.py @@ -4,17 +4,18 @@ # See top-level LICENSE file for more information from datetime import timedelta +from functools import partial import os -from os import path from typing import List, Tuple from unittest.mock import patch import pytest +from pytest_postgresql import factories import yaml -from swh.core.db.pytest_plugin import postgresql_fact -import swh.indexer +from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact from swh.indexer.storage import get_indexer_storage +from swh.indexer.storage.db import Db as IndexerDb from swh.objstorage.factory import get_objstorage from swh.storage import get_storage @@ -27,13 +28,19 @@ ] -SQL_FILES = path.join(path.dirname(swh.indexer.__file__), "sql", "*.sql") - - -idx_storage_postgresql = postgresql_fact( - "postgresql_proc", dbname="indexer_storage", dump_files=SQL_FILES, +idx_postgresql_proc = factories.postgresql_proc( + dbname="indexer_storage", + load=[ + partial( + initialize_database_for_module, + modname="indexer", + version=IndexerDb.current_version, + ) + ], ) +idx_storage_postgresql = postgresql_fact("idx_postgresql_proc") + @pytest.fixture def indexer_scheduler(swh_scheduler): diff --git a/swh/indexer/tests/storage/__init__.py b/swh/indexer/tests/storage/__init__.py --- a/swh/indexer/tests/storage/__init__.py +++ b/swh/indexer/tests/storage/__init__.py @@ -5,6 +5,6 @@ from os import path -import swh.storage +import swh.indexer SQL_DIR = path.join(path.dirname(swh.indexer.__file__), "sql") diff --git a/swh/indexer/tests/storage/conftest.py b/swh/indexer/tests/storage/conftest.py --- a/swh/indexer/tests/storage/conftest.py +++ b/swh/indexer/tests/storage/conftest.py @@ -9,8 +9,8 @@ from swh.indexer.storage import get_indexer_storage from swh.indexer.storage.model import ContentLicenseRow, ContentMimetypeRow +from swh.indexer.tests.conftest import idx_storage_postgresql from swh.model.hashutil import hash_to_bytes -from swh.storage.pytest_plugin import postgresql_fact from . import SQL_DIR from .generate_data_test import FOSSOLOGY_LICENSES, MIMETYPE_OBJECTS, TOOLS @@ -66,9 +66,7 @@ return (swh_indexer_storage, data) -swh_indexer_storage_postgresql = postgresql_fact( - "postgresql_proc", dump_files=DUMP_FILES -) +swh_indexer_storage_postgresql = idx_storage_postgresql @pytest.fixture