Page MenuHomeSoftware Heritage
Differential D1327

Make snapshot_add take an iterable of snapshot.
AbandonedPublic
Actions

Authored by vlorentz on Apr 1 2019, 4:12 PM.

Details

Reviewers
None
Group Reviewers
Reviewers
Summary

For uniformity with other _add() endpoints (except origin_visit_add).

Noone uses the new signature of snapshot_add except the tests yet,
so it's the perfect time to make this breaking change.

Diff Detail

Repository
rDSTO Storage manager
Branch
snapshot_add-many
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 4991
Build 6690: tox-on-jenkinsJenkins
Build 6689: arc lint + arc unit

Event Timeline

vlorentz created this revision.Apr 1 2019, 4:12 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptApr 1 2019, 4:12 PM
Harbormaster failed remote builds in B4990: Diff 4226!Apr 1 2019, 4:12 PM
swh-public-ci added a comment.Apr 1 2019, 4:12 PM

Build has FAILED

Link to build: https://jenkins.softwareheritage.org/job/DSTO/job/tox/262/
See console output for more information: https://jenkins.softwareheritage.org/job/DSTO/job/tox/262/console

vlorentz added a parent revision: D1325: Use sha1 instead of bigint as FK from origin_visit to snapshot (part 2: backfill + start using the new colum for reading + cleanup).Apr 1 2019, 4:13 PM
swh-public-ci added a comment.Apr 1 2019, 4:13 PM

Build has FAILED

Link to build: https://jenkins.softwareheritage.org/job/DSTO/job/tox/264/
See console output for more information: https://jenkins.softwareheritage.org/job/DSTO/job/tox/264/console

vlorentz updated this revision to Diff 4227.Apr 1 2019, 4:16 PM

re-run jenkins build

Harbormaster failed remote builds in B4991: Diff 4227!Apr 1 2019, 4:16 PM
swh-public-ci added a comment.Apr 1 2019, 4:16 PM

Build has FAILED

Link to build: https://jenkins.softwareheritage.org/job/DSTO/job/tox/265/
See console output for more information: https://jenkins.softwareheritage.org/job/DSTO/job/tox/265/console

vlorentz updated this revision to Diff 4228.Apr 1 2019, 4:21 PM
  • add SQL migration
Harbormaster failed remote builds in B4992: Diff 4228!Apr 1 2019, 4:21 PM
swh-public-ci added a comment.Apr 1 2019, 4:21 PM

Build has FAILED

Link to build: https://jenkins.softwareheritage.org/job/DSTO/job/tox/266/
See console output for more information: https://jenkins.softwareheritage.org/job/DSTO/job/tox/266/console

vlorentz abandoned this revision.Apr 1 2019, 4:21 PM

Revision Contents
Changeset List

PathSize
swh/
storage/
api/
10 lines
37 lines
sql/
1 line
54 lines
tests/
43 lines

Diff 4227

View Options

swh/storage/api/client.py

# Copyright (C) 2015-2022 The Software Heritage developers # Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import logging import logging
import os import os
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from psycopg2.errors import QueryCanceled from psycopg2.errors import OperationalError, QueryCanceled
from swh.core import config from swh.core import config
from swh.core.api import RPCServerApp from swh.core.api import RPCServerApp
from swh.core.api import encode_data_server as encode_data from swh.core.api import encode_data_server as encode_data
from swh.core.api import error_handler, serializers from swh.core.api import error_handler, serializers
from swh.storage import get_storage as get_swhstorage from swh.storage import get_storage as get_swhstorage
from ..exc import StorageArgumentException from ..exc import StorageArgumentException
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines
storage = None storage = None
@app.errorhandler(StorageArgumentException) @app.errorhandler(StorageArgumentException)
def argument_error_handler(exception): def argument_error_handler(exception):
return error_handler(exception, encode_data, status_code=400) return error_handler(exception, encode_data, status_code=400)
@app.errorhandler(QueryCanceled) @app.errorhandler(OperationalError)
def querycanceled_error_handler(exception): def operationalerror_exception_handler(exception):
# Same as error_handler(exception, encode_data); but does not log or send to Sentry. # Same as error_handler(exception, encode_data); but does not log or send to Sentry.
# These errors are noisy, and are better logged on the caller's side after it # These errors are noisy, and are better logged on the caller's side after it
# retried a few times # retried a few times.
# Additionally, we return 503 instead of 500, telling clients they should retry.
response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 503
return response
@app.errorhandler(QueryCanceled)
def querycancelled_exception_handler(exception):
# Ditto, but 500 instead of 503, because this is usually caused by the query
# size instead of a transient failure
response = encode_data(serializers.exception_to_dict(exception)) response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 500 response.status_code = 500
return response return response
@app.errorhandler(Exception) @app.errorhandler(Exception)
def default_error_handler(exception): def default_error_handler(exception):
return error_handler(exception, encode_data) return error_handler(exception, encode_data)
▲ Show 20 Lines • Show All 80 Lines • Show Last 20 Lines
View Options

swh/storage/in_memory.py

# Copyright (C) 2015-2022 The Software Heritage developers # Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import logging import logging
import os import os
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from psycopg2.errors import QueryCanceled from psycopg2.errors import OperationalError, QueryCanceled
from swh.core import config from swh.core import config
from swh.core.api import RPCServerApp from swh.core.api import RPCServerApp
from swh.core.api import encode_data_server as encode_data from swh.core.api import encode_data_server as encode_data
from swh.core.api import error_handler, serializers from swh.core.api import error_handler, serializers
from swh.storage import get_storage as get_swhstorage from swh.storage import get_storage as get_swhstorage
from ..exc import StorageArgumentException from ..exc import StorageArgumentException
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines
storage = None storage = None
@app.errorhandler(StorageArgumentException) @app.errorhandler(StorageArgumentException)
def argument_error_handler(exception): def argument_error_handler(exception):
return error_handler(exception, encode_data, status_code=400) return error_handler(exception, encode_data, status_code=400)
@app.errorhandler(QueryCanceled) @app.errorhandler(OperationalError)
def querycanceled_error_handler(exception): def operationalerror_exception_handler(exception):
# Same as error_handler(exception, encode_data); but does not log or send to Sentry. # Same as error_handler(exception, encode_data); but does not log or send to Sentry.
# These errors are noisy, and are better logged on the caller's side after it # These errors are noisy, and are better logged on the caller's side after it
# retried a few times # retried a few times.
# Additionally, we return 503 instead of 500, telling clients they should retry.
response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 503
return response
@app.errorhandler(QueryCanceled)
def querycancelled_exception_handler(exception):
# Ditto, but 500 instead of 503, because this is usually caused by the query
# size instead of a transient failure
response = encode_data(serializers.exception_to_dict(exception)) response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 500 response.status_code = 500
return response return response
@app.errorhandler(Exception) @app.errorhandler(Exception)
def default_error_handler(exception): def default_error_handler(exception):
return error_handler(exception, encode_data) return error_handler(exception, encode_data)
▲ Show 20 Lines • Show All 80 Lines • Show Last 20 Lines
View Options

swh/storage/sql/40-swh-func.sql

# Copyright (C) 2015-2022 The Software Heritage developers # Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import logging import logging
import os import os
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from psycopg2.errors import QueryCanceled from psycopg2.errors import OperationalError, QueryCanceled
from swh.core import config from swh.core import config
from swh.core.api import RPCServerApp from swh.core.api import RPCServerApp
from swh.core.api import encode_data_server as encode_data from swh.core.api import encode_data_server as encode_data
from swh.core.api import error_handler, serializers from swh.core.api import error_handler, serializers
from swh.storage import get_storage as get_swhstorage from swh.storage import get_storage as get_swhstorage
from ..exc import StorageArgumentException from ..exc import StorageArgumentException
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines
storage = None storage = None
@app.errorhandler(StorageArgumentException) @app.errorhandler(StorageArgumentException)
def argument_error_handler(exception): def argument_error_handler(exception):
return error_handler(exception, encode_data, status_code=400) return error_handler(exception, encode_data, status_code=400)
@app.errorhandler(QueryCanceled) @app.errorhandler(OperationalError)
def querycanceled_error_handler(exception): def operationalerror_exception_handler(exception):
# Same as error_handler(exception, encode_data); but does not log or send to Sentry. # Same as error_handler(exception, encode_data); but does not log or send to Sentry.
# These errors are noisy, and are better logged on the caller's side after it # These errors are noisy, and are better logged on the caller's side after it
# retried a few times # retried a few times.
# Additionally, we return 503 instead of 500, telling clients they should retry.
response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 503
return response
@app.errorhandler(QueryCanceled)
def querycancelled_exception_handler(exception):
# Ditto, but 500 instead of 503, because this is usually caused by the query
# size instead of a transient failure
response = encode_data(serializers.exception_to_dict(exception)) response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 500 response.status_code = 500
return response return response
@app.errorhandler(Exception) @app.errorhandler(Exception)
def default_error_handler(exception): def default_error_handler(exception):
return error_handler(exception, encode_data) return error_handler(exception, encode_data)
▲ Show 20 Lines • Show All 80 Lines • Show Last 20 Lines
View Options

swh/storage/storage.py

# Copyright (C) 2015-2022 The Software Heritage developers # Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import logging import logging
import os import os
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from psycopg2.errors import QueryCanceled from psycopg2.errors import OperationalError, QueryCanceled
from swh.core import config from swh.core import config
from swh.core.api import RPCServerApp from swh.core.api import RPCServerApp
from swh.core.api import encode_data_server as encode_data from swh.core.api import encode_data_server as encode_data
from swh.core.api import error_handler, serializers from swh.core.api import error_handler, serializers
from swh.storage import get_storage as get_swhstorage from swh.storage import get_storage as get_swhstorage
from ..exc import StorageArgumentException from ..exc import StorageArgumentException
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines
storage = None storage = None
@app.errorhandler(StorageArgumentException) @app.errorhandler(StorageArgumentException)
def argument_error_handler(exception): def argument_error_handler(exception):
return error_handler(exception, encode_data, status_code=400) return error_handler(exception, encode_data, status_code=400)
@app.errorhandler(QueryCanceled) @app.errorhandler(OperationalError)
def querycanceled_error_handler(exception): def operationalerror_exception_handler(exception):
# Same as error_handler(exception, encode_data); but does not log or send to Sentry. # Same as error_handler(exception, encode_data); but does not log or send to Sentry.
# These errors are noisy, and are better logged on the caller's side after it # These errors are noisy, and are better logged on the caller's side after it
# retried a few times # retried a few times.
# Additionally, we return 503 instead of 500, telling clients they should retry.
response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 503
return response
@app.errorhandler(QueryCanceled)
def querycancelled_exception_handler(exception):
# Ditto, but 500 instead of 503, because this is usually caused by the query
# size instead of a transient failure
response = encode_data(serializers.exception_to_dict(exception)) response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 500 response.status_code = 500
return response return response
@app.errorhandler(Exception) @app.errorhandler(Exception)
def default_error_handler(exception): def default_error_handler(exception):
return error_handler(exception, encode_data) return error_handler(exception, encode_data)
▲ Show 20 Lines • Show All 80 Lines • Show Last 20 Lines
View Options

swh/storage/tests/test_storage.py

# Copyright (C) 2015-2022 The Software Heritage developers # Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import logging import logging
import os import os
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from psycopg2.errors import QueryCanceled from psycopg2.errors import OperationalError, QueryCanceled
from swh.core import config from swh.core import config
from swh.core.api import RPCServerApp from swh.core.api import RPCServerApp
from swh.core.api import encode_data_server as encode_data from swh.core.api import encode_data_server as encode_data
from swh.core.api import error_handler, serializers from swh.core.api import error_handler, serializers
from swh.storage import get_storage as get_swhstorage from swh.storage import get_storage as get_swhstorage
from ..exc import StorageArgumentException from ..exc import StorageArgumentException
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines
storage = None storage = None
@app.errorhandler(StorageArgumentException) @app.errorhandler(StorageArgumentException)
def argument_error_handler(exception): def argument_error_handler(exception):
return error_handler(exception, encode_data, status_code=400) return error_handler(exception, encode_data, status_code=400)
@app.errorhandler(QueryCanceled) @app.errorhandler(OperationalError)
def querycanceled_error_handler(exception): def operationalerror_exception_handler(exception):
# Same as error_handler(exception, encode_data); but does not log or send to Sentry. # Same as error_handler(exception, encode_data); but does not log or send to Sentry.
# These errors are noisy, and are better logged on the caller's side after it # These errors are noisy, and are better logged on the caller's side after it
# retried a few times # retried a few times.
# Additionally, we return 503 instead of 500, telling clients they should retry.
response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 503
return response
@app.errorhandler(QueryCanceled)
def querycancelled_exception_handler(exception):
# Ditto, but 500 instead of 503, because this is usually caused by the query
# size instead of a transient failure
response = encode_data(serializers.exception_to_dict(exception)) response = encode_data(serializers.exception_to_dict(exception))
response.status_code = 500 response.status_code = 500
return response return response
@app.errorhandler(Exception) @app.errorhandler(Exception)
def default_error_handler(exception): def default_error_handler(exception):
return error_handler(exception, encode_data) return error_handler(exception, encode_data)
▲ Show 20 Lines • Show All 80 Lines • Show Last 20 Lines
About · Developer information · Legal