Page MenuHomeSoftware Heritage
Differential D3909

cli: speedup the `swh` cli command startup time
ClosedPublic
Actions

Authored by douardda on Sep 10 2020, 4:09 PM.

Details

Reviewers
vlorentz
Group Reviewers
Reviewers
Maniphest Tasks
T2575: Investigate if/how we could improve `swh` cli command startup time
Commits
rDDEPde0ab7f6f21f: cli: speedup the `swh` cli command startup time
Summary

by moving import statements in functions.

Related to T2575.

Diff Detail

Repository
rDDEP Push deposit
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

douardda created this revision.Sep 10 2020, 4:09 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptSep 10 2020, 4:09 PM
swh-public-ci added a comment.Sep 10 2020, 4:11 PM

Build has FAILED

Patch application report for D3909 (id=13768)

Rebasing onto 7c5c84b595...

Current branch diff-target is up to date.
Changes applied before test
commit f8d9a8404572abed76bd8a1dcb7cb11a6c455d9f
Author: David Douard <david.douard@sdfa3.org>
Date:   Wed Sep 9 18:22:50 2020 +0200

    cli: speedup the `swh` cli command startup time
    
    by moving import statements in functions.
    
    Related to T2575.

Link to build: https://jenkins.softwareheritage.org/job/DDEP/job/tests-on-diff/62/
See console output for more information: https://jenkins.softwareheritage.org/job/DDEP/job/tests-on-diff/62/console

Harbormaster failed remote builds in B14995: Diff 13768!Sep 10 2020, 4:11 PM
vlorentz accepted this revision.Sep 10 2020, 4:41 PM
This revision is now accepted and ready to land.Sep 10 2020, 4:41 PM
douardda updated this revision to Diff 13789.Sep 10 2020, 5:10 PM

fix test_cli

broken by (moved) mocked objects.

swh-public-ci added a comment.Sep 10 2020, 5:12 PM

Build is green

Patch application report for D3909 (id=13789)

Rebasing onto 7c5c84b595...

Current branch diff-target is up to date.
Changes applied before test
commit de0ab7f6f21fb26fb0e59c16eeb5117d22ada2fd
Author: David Douard <david.douard@sdfa3.org>
Date:   Wed Sep 9 18:22:50 2020 +0200

    cli: speedup the `swh` cli command startup time
    
    by moving import statements in functions.
    
    Related to T2575.

See https://jenkins.softwareheritage.org/job/DDEP/job/tests-on-diff/63/ for more details.

Harbormaster completed remote builds in B15015: Diff 13789.Sep 10 2020, 5:12 PM
Closed by commit rDDEPde0ab7f6f21f: cli: speedup the `swh` cli command startup time (authored by douardda). · Explain WhySep 10 2020, 5:17 PM
This revision was automatically updated to reflect the committed changes.
douardda added a commit: rDDEPde0ab7f6f21f: cli: speedup the `swh` cli command startup time.

Revision Contents
Changeset List

PathSize
swh/
deposit/
cli/
2 lines
5 lines
22 lines
tests/
cli/
6 lines

Diff 13802

View Options

swh/deposit/cli/__init__.py

# Copyright (C) 2017-2019 The Software Heritage developers # Copyright (C) 2017-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
# WARNING: do not import unnecessary things here to keep cli startup time under
# control
import click import click
import logging import logging
from swh.core.cli import CONTEXT_SETTINGS from swh.core.cli import CONTEXT_SETTINGS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
Show All 27 Lines
View Options

swh/deposit/cli/admin.py

# Copyright (C) 2017-2020 The Software Heritage developers # Copyright (C) 2017-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
# WARNING: do not import unnecessary things here to keep cli startup time under
# control
import click import click
from swh.deposit.config import setup_django_for
from swh.deposit.cli import deposit from swh.deposit.cli import deposit
@deposit.group("admin") @deposit.group("admin")
@click.option( @click.option(
"--config-file", "--config-file",
"-C", "-C",
default=None, default=None,
type=click.Path(exists=True, dir_okay=False,), type=click.Path(exists=True, dir_okay=False,),
help="Optional extra configuration file.", help="Optional extra configuration file.",
) )
@click.option( @click.option(
"--platform", "--platform",
default="development", default="development",
type=click.Choice(["development", "production"]), type=click.Choice(["development", "production"]),
help="development or production platform", help="development or production platform",
) )
@click.pass_context @click.pass_context
def admin(ctx, config_file, platform): def admin(ctx, config_file, platform):
"""Server administration tasks (manipulate user or collections)""" """Server administration tasks (manipulate user or collections)"""
from swh.deposit.config import setup_django_for
# configuration happens here # configuration happens here
setup_django_for(platform, config_file=config_file) setup_django_for(platform, config_file=config_file)
@admin.group("user") @admin.group("user")
@click.pass_context @click.pass_context
def user(ctx): def user(ctx):
"""Manipulate user.""" """Manipulate user."""
▲ Show 20 Lines • Show All 235 Lines • Show Last 20 Lines
View Options

swh/deposit/cli/client.py

# Copyright (C) 2017-2020 The Software Heritage developers # Copyright (C) 2017-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
# WARNING: do not import unnecessary things here to keep cli startup time under
# control
import os import os
import logging import logging
import sys import sys
import tempfile
import uuid
import json
import yaml
import click import click
import xmltodict
from swh.deposit.client import PublicApiDepositClient, MaintenanceError
from swh.deposit.cli import deposit from swh.deposit.cli import deposit
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class InputError(ValueError): class InputError(ValueError):
"""Input script error """Input script error
""" """
pass pass
def generate_slug(): def generate_slug():
"""Generate a slug (sample purposes). """Generate a slug (sample purposes).
""" """
import uuid
return str(uuid.uuid4()) return str(uuid.uuid4())
def _url(url): def _url(url):
"""Force the /1 api version at the end of the url (avoiding confusing """Force the /1 api version at the end of the url (avoiding confusing
issues without it). issues without it).
Args: Args:
Show All 21 Lines Args:
name (str): Software's name name (str): Software's name
external_id (str): External identifier (slug) or generated one external_id (str): External identifier (slug) or generated one
authors (List[str]): List of author names authors (List[str]): List of author names
Returns: Returns:
Filepath to the metadata generated file Filepath to the metadata generated file
""" """
import xmltodict
path = os.path.join(temp_dir, "metadata.xml") path = os.path.join(temp_dir, "metadata.xml")
# generate a metadata file with the minimum required metadata # generate a metadata file with the minimum required metadata
codemetadata = { codemetadata = {
"entry": { "entry": {
"@xmlns": "http://www.w3.org/2005/Atom", "@xmlns": "http://www.w3.org/2005/Atom",
"@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", "@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0",
"codemeta:name": name, "codemeta:name": name,
"codemeta:identifier": external_id, "codemeta:identifier": external_id,
Show All 16 Linesdef _client(url, username, password):
"""Instantiate a client to access the deposit api server """Instantiate a client to access the deposit api server
Args: Args:
url (str): Deposit api server url (str): Deposit api server
username (str): User username (str): User
password (str): User's password password (str): User's password
""" """
from swh.deposit.client import PublicApiDepositClient
client = PublicApiDepositClient( client = PublicApiDepositClient(
{"url": url, "auth": {"username": username, "password": password},} {"url": url, "auth": {"username": username, "password": password},}
) )
return client return client
def _collection(client): def _collection(client):
"""Retrieve the client's collection """Retrieve the client's collection
▲ Show 20 Lines • Show All 285 Lines • ▼ Show 20 Lines):
"""Software Heritage Public Deposit Client """Software Heritage Public Deposit Client
Create/Update deposit through the command line. Create/Update deposit through the command line.
More documentation can be found at More documentation can be found at
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html. https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html.
""" """
import tempfile
from swh.deposit.client import MaintenanceError
url = _url(url) url = _url(url)
config = {} config = {}
with tempfile.TemporaryDirectory() as temp_dir: with tempfile.TemporaryDirectory() as temp_dir:
try: try:
logger.debug("Parsing cli options") logger.debug("Parsing cli options")
config = client_command_parse_input( config = client_command_parse_input(
username, username,
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines@click.option(
type=click.Choice(["logging", "yaml", "json"]), type=click.Choice(["logging", "yaml", "json"]),
help="Output format results.", help="Output format results.",
) )
@click.pass_context @click.pass_context
def status(ctx, url, username, password, deposit_id, output_format): def status(ctx, url, username, password, deposit_id, output_format):
"""Deposit's status """Deposit's status
""" """
from swh.deposit.client import MaintenanceError
url = _url(url) url = _url(url)
logger.debug("Status deposit") logger.debug("Status deposit")
try: try:
client = _client(url, username, password) client = _client(url, username, password)
collection = _collection(client) collection = _collection(client)
except InputError as e: except InputError as e:
logger.error("Problem during parsing options: %s", e) logger.error("Problem during parsing options: %s", e)
sys.exit(1) sys.exit(1)
except MaintenanceError as e: except MaintenanceError as e:
logger.error(e) logger.error(e)
sys.exit(1) sys.exit(1)
print_result( print_result(
client.deposit_status(collection=collection, deposit_id=deposit_id), client.deposit_status(collection=collection, deposit_id=deposit_id),
output_format, output_format,
) )
def print_result(data, output_format): def print_result(data, output_format):
import json
import yaml
if output_format == "json": if output_format == "json":
click.echo(json.dumps(data)) click.echo(json.dumps(data))
elif output_format == "yaml": elif output_format == "yaml":
click.echo(yaml.dump(data)) click.echo(yaml.dump(data))
else: else:
logger.info(data) logger.info(data)
View Options

swh/deposit/tests/cli/test_client.py

Show First 20 Lines • Show All 138 Lines • ▼ Show 20 Linesdef test_single_minimal_deposit(
sample_archive, mocker, caplog, client_mock, slug, tmp_path sample_archive, mocker, caplog, client_mock, slug, tmp_path
): ):
""" from: """ from:
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit
""" # noqa """ # noqa
metadata_path = os.path.join(tmp_path, "metadata.xml") metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch( mocker.patch(
"swh.deposit.cli.client.tempfile.TemporaryDirectory", "tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)), return_value=contextlib.nullcontext(str(tmp_path)),
) )
runner = CliRunner() runner = CliRunner()
result = runner.invoke( result = runner.invoke(
cli, cli,
[ [
"upload", "upload",
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Linesdef test_metadata_validation(sample_archive, mocker, caplog, tmp_path):
mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug) mocker.patch("swh.deposit.cli.client.generate_slug", return_value=slug)
mock_client = MagicMock() mock_client = MagicMock()
mocker.patch("swh.deposit.cli.client._client", return_value=mock_client) mocker.patch("swh.deposit.cli.client._client", return_value=mock_client)
mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT mock_client.service_document.return_value = EXAMPLE_SERVICE_DOCUMENT
mock_client.deposit_create.return_value = '{"foo": "bar"}' mock_client.deposit_create.return_value = '{"foo": "bar"}'
metadata_path = os.path.join(tmp_path, "metadata.xml") metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch( mocker.patch(
"swh.deposit.cli.client.tempfile.TemporaryDirectory", "tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)), return_value=contextlib.nullcontext(str(tmp_path)),
) )
with open(metadata_path, "a"): with open(metadata_path, "a"):
pass # creates the file pass # creates the file
runner = CliRunner() runner = CliRunner()
# Test missing author # Test missing author
▲ Show 20 Lines • Show All 92 Lines • ▼ Show 20 Lines):
""" from: """ from:
https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit https://docs.softwareheritage.org/devel/swh-deposit/getting-started.html#single-deposit
""" # noqa """ # noqa
slug = "my-slug" slug = "my-slug"
collection = "my-collection" collection = "my-collection"
metadata_path = os.path.join(tmp_path, "metadata.xml") metadata_path = os.path.join(tmp_path, "metadata.xml")
mocker.patch( mocker.patch(
"swh.deposit.cli.client.tempfile.TemporaryDirectory", "tempfile.TemporaryDirectory",
return_value=contextlib.nullcontext(str(tmp_path)), return_value=contextlib.nullcontext(str(tmp_path)),
) )
runner = CliRunner() runner = CliRunner()
result = runner.invoke( result = runner.invoke(
cli, cli,
[ [
"upload", "upload",
▲ Show 20 Lines • Show All 132 Lines • Show Last 20 Lines
About · Developer information · Legal