diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ version.txt swh/lister/_version.py .tox/ +.mypy_cache/ diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,3 +7,4 @@ include swh/lister/cran/list_all_packages.R recursive-include swh/lister/*/tests/ *.json *.html *.txt *.* * recursive-include swh/lister/cgit/tests/data/ *.* * +recursive-include swh py.typed diff --git a/mypy.ini b/mypy.ini new file mode 100644 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,39 @@ +[mypy] +namespace_packages = True +warn_unused_ignores = True + +# support for sqlalchemy magic: see https://github.com/dropbox/sqlalchemy-stubs +plugins = sqlmypy + + +# 3rd party libraries without stubs (yet) + +[mypy-bs4.*] +ignore_missing_imports = True + +[mypy-celery.*] +ignore_missing_imports = True + +[mypy-debian.*] +ignore_missing_imports = True + +[mypy-iso8601.*] +ignore_missing_imports = True + +[mypy-pkg_resources.*] +ignore_missing_imports = True + +[mypy-pytest.*] +ignore_missing_imports = True + +[mypy-requests_mock.*] +ignore_missing_imports = True + +[mypy-testing.postgresql.*] +ignore_missing_imports = True + +[mypy-urllib3.util.*] +ignore_missing_imports = True + +[mypy-xmltodict.*] +ignore_missing_imports = True diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,3 +2,4 @@ pytest-postgresql requests_mock testing.postgresql +sqlalchemy-stubs diff --git a/swh/__init__.py b/swh/__init__.py --- a/swh/__init__.py +++ b/swh/__init__.py @@ -1 +1,4 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +from pkgutil import extend_path +from typing import Iterable + +__path__ = extend_path(__path__, __name__) # type: Iterable[str] diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -6,6 +6,7 @@ import iso8601 from datetime import datetime, timezone +from typing import Any from urllib import parse from swh.lister.bitbucket.models import BitBucketModel @@ -21,7 +22,7 @@ LISTER_NAME = 'bitbucket' DEFAULT_URL = 'https://api.bitbucket.org/2.0' instance = 'bitbucket' - default_min_bound = datetime.fromtimestamp(0, timezone.utc) + default_min_bound = datetime.fromtimestamp(0, timezone.utc) # type: Any def __init__(self, url=None, override_config=None, per_page=100): super().__init__(url=url, override_config=override_config) diff --git a/swh/lister/bitbucket/tests/test_bb_lister.py b/swh/lister/bitbucket/tests/test_bb_lister.py --- a/swh/lister/bitbucket/tests/test_bb_lister.py +++ b/swh/lister/bitbucket/tests/test_bb_lister.py @@ -6,7 +6,6 @@ import unittest from datetime import timedelta - from urllib.parse import unquote import iso8601 @@ -16,7 +15,7 @@ from swh.lister.core.tests.test_lister import HttpListerTester -def convert_type(req_index): +def _convert_type(req_index): """Convert the req_index to its right type according to the model's "indexable" column. @@ -30,17 +29,17 @@ lister_subdir = 'bitbucket' good_api_response_file = 'api_response.json' bad_api_response_file = 'api_empty_response.json' - first_index = convert_type('2008-07-12T07:44:01.476818+00:00') - last_index = convert_type('2008-07-19T06:16:43.044743+00:00') + first_index = _convert_type('2008-07-12T07:44:01.476818+00:00') + last_index = _convert_type('2008-07-19T06:16:43.044743+00:00') entries_per_page = 10 - convert_type = staticmethod(convert_type) + convert_type = _convert_type def request_index(self, request): """(Override) This is needed to emulate the listing bootstrap when no min_bound is provided to run """ m = self.test_re.search(request.path_url) - idx = convert_type(m.group(1)) + idx = _convert_type(m.group(1)) if idx == self.Lister.default_min_bound: idx = self.first_index return idx diff --git a/swh/lister/core/abstractattribute.py b/swh/lister/core/abstractattribute.py --- a/swh/lister/core/abstractattribute.py +++ b/swh/lister/core/abstractattribute.py @@ -16,7 +16,8 @@ import abc class ClassContainingAnAbstractAttribute(abc.ABC): - foo = AbstractAttribute('descriptive docstring for foo') + foo: Union[AbstractAttribute, Any] = \ + AbstractAttribute('docstring for foo') """ __isabstractmethod__ = True diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -13,6 +13,7 @@ from sqlalchemy import create_engine, func from sqlalchemy.orm import sessionmaker +from typing import Any, Type, Union from swh.core import config from swh.scheduler import get_scheduler, utils @@ -64,10 +65,12 @@ def is_within_bounds """ - MODEL = AbstractAttribute('Subclass type (not instance)' - ' of swh.lister.core.models.ModelBase' - ' customized for a specific service.') - LISTER_NAME = AbstractAttribute("Lister's name") + MODEL = AbstractAttribute( + 'Subclass type (not instance) of swh.lister.core.models.ModelBase ' + 'customized for a specific service.' + ) # type: Union[AbstractAttribute, Type[Any]] + LISTER_NAME = AbstractAttribute( + "Lister's name") # type: Union[AbstractAttribute, str] def transport_request(self, identifier): """Given a target endpoint identifier to query, try once to request it. diff --git a/swh/lister/core/lister_transports.py b/swh/lister/core/lister_transports.py --- a/swh/lister/core/lister_transports.py +++ b/swh/lister/core/lister_transports.py @@ -12,6 +12,8 @@ import requests import xmltodict +from typing import Optional, Union + try: from swh.lister._version import __version__ except ImportError: @@ -29,14 +31,14 @@ To be used in conjunction with ListerBase or a subclass of it. """ - DEFAULT_URL = None - PATH_TEMPLATE = AbstractAttribute('string containing a python string' - ' format pattern that produces the API' - ' endpoint path for listing stored' - ' repositories when given an index.' - ' eg. "/repositories?after=%s".' - 'To be implemented in the API-specific' - ' class inheriting this.') + DEFAULT_URL = None # type: Optional[str] + PATH_TEMPLATE = \ + AbstractAttribute( + 'string containing a python string format pattern that produces' + ' the API endpoint path for listing stored repositories when given' + ' an index, e.g., "/repositories?after=%s". To be implemented in' + ' the API-specific class inheriting this.' + ) # type: Union[AbstractAttribute, Optional[str]] EXPECTED_STATUS_CODES = (200, 429, 403, 404) @@ -214,8 +216,9 @@ To be used in conjunction with ListerBase or a subclass of it. """ - PAGE = AbstractAttribute("The server api's unique page to retrieve and " - "parse for information") + PAGE = AbstractAttribute( + "URL of the API's unique page to retrieve and parse " + "for information") # type: Union[AbstractAttribute, str] PATH_TEMPLATE = None # we do not use it def __init__(self, url=None): diff --git a/swh/lister/core/models.py b/swh/lister/core/models.py --- a/swh/lister/core/models.py +++ b/swh/lister/core/models.py @@ -8,6 +8,7 @@ from sqlalchemy import Column, DateTime, Integer, String from sqlalchemy.ext.declarative import DeclarativeMeta +from typing import Type, Union from .abstractattribute import AbstractAttribute @@ -24,9 +25,12 @@ class ModelBase(SQLBase, metaclass=ABCSQLMeta): """a common repository""" __abstract__ = True - __tablename__ = AbstractAttribute + __tablename__ = \ + AbstractAttribute # type: Union[Type[AbstractAttribute], str] - uid = AbstractAttribute('Column(, primary_key=True)') + uid = AbstractAttribute( + 'Column(, primary_key=True)' + ) # type: Union[AbstractAttribute, Column] name = Column(String, index=True) full_name = Column(String, index=True) @@ -45,11 +49,14 @@ class IndexingModelBase(ModelBase, metaclass=ABCSQLMeta): __abstract__ = True - __tablename__ = AbstractAttribute + __tablename__ = \ + AbstractAttribute # type: Union[Type[AbstractAttribute], str] # The value used for sorting, segmenting, or api query paging, # because uids aren't always sequential. - indexable = AbstractAttribute('Column(, index=True)') + indexable = AbstractAttribute( + 'Column(, index=True)' + ) # type: Union[AbstractAttribute, Column] def initialize(db_engine, drop_tables=False, **kwargs): diff --git a/swh/lister/core/tests/test_abstractattribute.py b/swh/lister/core/tests/test_abstractattribute.py --- a/swh/lister/core/tests/test_abstractattribute.py +++ b/swh/lister/core/tests/test_abstractattribute.py @@ -5,13 +5,15 @@ import abc import unittest +from typing import Any + from swh.lister.core.abstractattribute import AbstractAttribute class BaseClass(abc.ABC): - v1 = AbstractAttribute - v2 = AbstractAttribute() - v3 = AbstractAttribute('changed docstring') + v1 = AbstractAttribute # type: Any + v2 = AbstractAttribute() # type: Any + v3 = AbstractAttribute('changed docstring') # type: Any v4 = 'qux' diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py --- a/swh/lister/core/tests/test_lister.py +++ b/swh/lister/core/tests/test_lister.py @@ -10,6 +10,7 @@ import requests_mock from sqlalchemy import create_engine +from typing import Any, Callable, Optional, Pattern, Type, Union from swh.lister.core.abstractattribute import AbstractAttribute from swh.lister.tests.test_utils import init_db @@ -28,9 +29,12 @@ to customize for a specific listing service. """ - Lister = AbstractAttribute('The lister class to test') - lister_subdir = AbstractAttribute('bitbucket, github, etc.') - good_api_response_file = AbstractAttribute('Example good response body') + Lister = AbstractAttribute( + 'Lister class to test') # type: Union[AbstractAttribute, Type[Any]] + lister_subdir = AbstractAttribute( + 'bitbucket, github, etc.') # type: Union[AbstractAttribute, str] + good_api_response_file = AbstractAttribute( + 'Example good response body') # type: Union[AbstractAttribute, str] LISTER_NAME = 'fake-lister' # May need to override this if the headers are used for something @@ -157,13 +161,21 @@ to customize for a specific listing service. """ - last_index = AbstractAttribute('Last index in good_api_response') - first_index = AbstractAttribute('First index in good_api_response') - bad_api_response_file = AbstractAttribute('Example bad response body') - entries_per_page = AbstractAttribute('Number of results in good response') - test_re = AbstractAttribute('Compiled regex matching the server url. Must' - ' capture the index value.') - convert_type = str + last_index = AbstractAttribute( + 'Last index ' + 'in good_api_response') # type: Union[AbstractAttribute, int] + first_index = AbstractAttribute( + 'First index in ' + ' good_api_response') # type: Union[AbstractAttribute, Optional[int]] + bad_api_response_file = AbstractAttribute( + 'Example bad response body') # type: Union[AbstractAttribute, str] + entries_per_page = AbstractAttribute( + 'Number of results in ' + 'good response') # type: Union[AbstractAttribute, int] + test_re = AbstractAttribute( + 'Compiled regex matching the server url. Must capture the ' + 'index value.') # type: Union[AbstractAttribute, Pattern] + convert_type = str # type: Callable[..., Any] """static method used to convert the "request_index" to its right type (for indexing listers for example, this is in accordance with the model's "indexable" column). @@ -343,9 +355,12 @@ to customize for a specific listing service. """ - entries = AbstractAttribute('Number of results in good response') - PAGE = AbstractAttribute("The server api's unique page to retrieve and " - "parse for information") + entries = AbstractAttribute( + 'Number of results ' + 'in good response') # type: Union[AbstractAttribute, int] + PAGE = AbstractAttribute( + "URL of the server api's unique page to retrieve and " + "parse for information") # type: Union[AbstractAttribute, str] def get_fl(self, override_config=None): """Retrieve an instance of fake lister (fl). diff --git a/swh/lister/cran/lister.py b/swh/lister/cran/lister.py --- a/swh/lister/cran/lister.py +++ b/swh/lister/cran/lister.py @@ -8,7 +8,7 @@ import subprocess from collections import defaultdict -from typing import List, Dict +from typing import Any, Dict, List, Mapping from swh.lister.cran.models import CRANModel @@ -23,7 +23,7 @@ MODEL = CRANModel LISTER_NAME = 'cran' instance = 'cran' - descriptions = defaultdict(dict) + descriptions = defaultdict(dict) # type: Mapping[str, Mapping[Any, Any]] def task_dict(self, origin_type, origin_url, **kwargs): """Return task format dict diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py --- a/swh/lister/github/lister.py +++ b/swh/lister/github/lister.py @@ -5,6 +5,8 @@ import re import time +from typing import Any + from swh.lister.core.indexing_lister import IndexingHttpLister from swh.lister.github.models import GitHubModel @@ -16,7 +18,7 @@ API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)') LISTER_NAME = 'github' instance = 'github' # There is only 1 instance of such lister - default_min_bound = 0 + default_min_bound = 0 # type: Any def get_model_from_repo(self, repo): return { diff --git a/swh/lister/phabricator/lister.py b/swh/lister/phabricator/lister.py --- a/swh/lister/phabricator/lister.py +++ b/swh/lister/phabricator/lister.py @@ -20,7 +20,8 @@ class PhabricatorLister(IndexingHttpLister): PATH_TEMPLATE = '?order=oldest&attachments[uris]=1&after=%s' - DEFAULT_URL = 'https://forge.softwareheritage.org/api/diffusion.repository.search' # noqa + DEFAULT_URL = \ + 'https://forge.softwareheritage.org/api/diffusion.repository.search' MODEL = PhabricatorModel LISTER_NAME = 'phabricator' diff --git a/swh/lister/py.typed b/swh/lister/py.typed new file mode 100644 --- /dev/null +++ b/swh/lister/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=flake8,py3 +envlist=flake8,mypy,py3 [testenv:py3] deps = @@ -17,3 +17,11 @@ flake8 commands = {envpython} -m flake8 + +[testenv:mypy] +skip_install = true +deps = + .[testing] + mypy +commands = + mypy swh