diff --git a/.gitignore b/.gitignore
index f5fc2ae..21e2c07 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,11 @@
*.pyc
*.sw?
*~
.coverage
.eggs/
__pycache__
*.egg-info/
-version.txt
\ No newline at end of file
+version.txt
+build/
+dist/
+.tox
diff --git a/PKG-INFO b/PKG-INFO
index e9f3971..b04d0da 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,10 +1,136 @@
-Metadata-Version: 1.0
+Metadata-Version: 2.1
Name: swh.loader.pypi
-Version: 0.0.2
+Version: 0.0.3
Summary: Software Heritage PyPI Loader
-Home-page: https://forge.softwareheritage.org/source/swh-loader-pypi.git
+Home-page: https://forge.softwareheritage.org/source/swh-loader-pypi
Author: Software Heritage developers
Author-email: swh-devel@inria.fr
License: UNKNOWN
-Description: UNKNOWN
+Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
+Project-URL: Funding, https://www.softwareheritage.org/donate
+Project-URL: Source, https://forge.softwareheritage.org/source/swh-loader-pypi
+Description: swh-loader-pypi
+ ====================
+
+ SWH PyPI loader's source code repository
+
+ # What does the loader do?
+
+ The PyPI loader visits and loads a PyPI project [1].
+
+ Each visit will result in:
+ - 1 snapshot (which targets n revisions ; 1 per release artifact)
+ - 1 revision (which targets 1 directory ; the release artifact uncompressed)
+
+ [1] https://pypi.org/help/#packages
+
+ ## First visit
+
+ Given a PyPI project (origin), the loader, for the first visit:
+
+ - retrieves information for the given project (including releases)
+ - then for each associated release
+ - for each associated source distribution (type 'sdist') release
+ artifact (possibly many per release)
+ - retrieves the associated artifact archive (with checks)
+ - uncompresses locally the archive
+ - computes the hashes of the uncompressed directory
+ - then creates a revision (using PKG-INFO metadata file) targeting
+ such directory
+ - finally, creates a snapshot targeting all seen revisions
+ (uncompressed PyPI artifact and metadata).
+
+ ## Next visit
+
+ The loader starts by checking if something changed since the last
+ visit. If nothing changed, the visit's snapshot is left
+ unchanged. The new visit targets the same snapshot.
+
+ If something changed, the already seen release artifacts are skipped.
+ Only the new ones are loaded. In the end, the loader creates a new
+ snapshot based on the previous one. Thus, the new snapshot targets
+ both the old and new PyPI release artifacts.
+
+ ## Terminology
+
+ - 1 project: a PyPI project (used as swh origin). This is a collection
+ of releases.
+
+ - 1 release: a specific version of the (PyPi) project. It's a
+ collection of information and associated source release
+ artifacts (type 'sdist')
+
+ - 1 release artifact: a source release artifact (distributed by a PyPI
+ maintainer). In swh, we are specifically
+ interested by the 'sdist' type (source code).
+
+ ## Edge cases
+
+ - If no release provides release artifacts, those are skipped
+
+ - If a release artifact holds no PKG-INFO file (root at the archive),
+ the release artifact is skipped.
+
+ - If a problem occurs during a fetch action (e.g. release artifact
+ download), the load fails and the visit is marked as 'partial'.
+
+ # Development
+
+ ## Configuration file
+
+ ### Location
+
+ Either:
+ - /etc/softwareheritage/
+ - ~/.config/swh/
+ - ~/.swh/
+
+ Note: Will call that location $SWH_CONFIG_PATH
+
+ ### Configuration sample
+
+ $SWH_CONFIG_PATH/loader/pypi.yml:
+ ```
+ storage:
+ cls: remote
+ args:
+ url: http://localhost:5002/
+
+ ```
+
+ ## Local run
+
+ The built-in command-line will run the loader for a project in the
+ main PyPI archive.
+
+ For instance, to load arrow:
+ ``` sh
+ python3 -m swh.loader.pypi.loader arrow
+ ```
+
+ If you need more control, you can use the loader directly. It expects
+ three arguments:
+ - project: a PyPI project name (f.e.: arrow)
+ - project_url: URL of the PyPI project (human-readable html page)
+ - project_metadata_url: URL of the PyPI metadata information
+ (machine-parsable json document)
+
+ ``` python
+ import logging
+ logging.basicConfig(level=logging.DEBUG)
+
+ from swh.loader.pypi.tasks import LoadPyPI
+
+ project='arrow'
+
+ LoadPyPI().run(project, 'https://pypi.org/pypi/%s/' % project, 'https://pypi.org/pypi/%s/json' % project)
+ ```
+
Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Description-Content-Type: text/markdown
+Provides-Extra: testing
diff --git a/README.md b/README.md
index 38457ab..1f0cdc2 100644
--- a/README.md
+++ b/README.md
@@ -1,109 +1,116 @@
swh-loader-pypi
====================
SWH PyPI loader's source code repository
# What does the loader do?
The PyPI loader visits and loads a PyPI project [1].
Each visit will result in:
- 1 snapshot (which targets n revisions ; 1 per release artifact)
- 1 revision (which targets 1 directory ; the release artifact uncompressed)
[1] https://pypi.org/help/#packages
## First visit
Given a PyPI project (origin), the loader, for the first visit:
- retrieves information for the given project (including releases)
- then for each associated release
- for each associated source distribution (type 'sdist') release
artifact (possibly many per release)
- retrieves the associated artifact archive (with checks)
- uncompresses locally the archive
- computes the hashes of the uncompressed directory
-- then creates a revision (using PKG-INFO metadata file)
- targetting such directory
-- finally, creates a snapshot targetting all seen revisions
+- then creates a revision (using PKG-INFO metadata file) targeting
+ such directory
+- finally, creates a snapshot targeting all seen revisions
(uncompressed PyPI artifact and metadata).
## Next visit
The loader starts by checking if something changed since the last
visit. If nothing changed, the visit's snapshot is left
unchanged. The new visit targets the same snapshot.
If something changed, the already seen release artifacts are skipped.
Only the new ones are loaded. In the end, the loader creates a new
snapshot based on the previous one. Thus, the new snapshot targets
both the old and new PyPI release artifacts.
## Terminology
- 1 project: a PyPI project (used as swh origin). This is a collection
of releases.
- 1 release: a specific version of the (PyPi) project. It's a
collection of information and associated source release
artifacts (type 'sdist')
- 1 release artifact: a source release artifact (distributed by a PyPI
maintainer). In swh, we are specifically
interested by the 'sdist' type (source code).
## Edge cases
- If no release provides release artifacts, those are skipped
- If a release artifact holds no PKG-INFO file (root at the archive),
the release artifact is skipped.
- If a problem occurs during a fetch action (e.g. release artifact
download), the load fails and the visit is marked as 'partial'.
# Development
## Configuration file
### Location
Either:
-- /etc/softwareheritage/loader/pypi.yml
-- ~/.config/swh/loader/pypi.yml
-- ~/.swh/loader/svn.pypi
+- /etc/softwareheritage/
+- ~/.config/swh/
+- ~/.swh/
+
+Note: Will call that location $SWH_CONFIG_PATH
### Configuration sample
+$SWH_CONFIG_PATH/loader/pypi.yml:
```
storage:
cls: remote
args:
url: http://localhost:5002/
```
## Local run
-The built-in command-line will run the loader for a project in the main PyPI archive.
+The built-in command-line will run the loader for a project in the
+main PyPI archive.
+
For instance, to load arrow:
``` sh
-python3 -m swh.loader.pypi arrow
+python3 -m swh.loader.pypi.loader arrow
```
-If you need more control, you can use the loader directly. It expects three arguments:
+If you need more control, you can use the loader directly. It expects
+three arguments:
- project: a PyPI project name (f.e.: arrow)
- project_url: URL of the PyPI project (human-readable html page)
-- project_metadata_url: URL of the PyPI metadata information (machine-parsable json document)
+- project_metadata_url: URL of the PyPI metadata information
+ (machine-parsable json document)
``` python
import logging
logging.basicConfig(level=logging.DEBUG)
from swh.loader.pypi.tasks import LoadPyPI
project='arrow'
LoadPyPI().run(project, 'https://pypi.org/pypi/%s/' % project, 'https://pypi.org/pypi/%s/json' % project)
```
diff --git a/docs/index.rst b/docs/index.rst
index 8b64117..a1468c6 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,15 +1,19 @@
-Software Heritage - Development Documentation
-=============================================
+.. _swh-loader-pypi:
+
+Software Heritage - PyPI loader
+===============================
+
+Loader for `PyPI `_ source code releases.
+
.. toctree::
:maxdepth: 2
:caption: Contents:
-
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
diff --git a/requirements-swh.txt b/requirements-swh.txt
index 7274d66..7f8da48 100644
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1,5 +1,5 @@
swh.core
swh.model >= 0.0.27
swh.storage
swh.scheduler
-swh.loader.core
+swh.loader.core >= 0.0.34
diff --git a/setup.py b/setup.py
index 033ec00..fe8a048 100755
--- a/setup.py
+++ b/setup.py
@@ -1,40 +1,66 @@
#!/usr/bin/env python3
-
-import os
+# Copyright (C) 2015-2018 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
from setuptools import setup, find_packages
+from os import path
+from io import open
+
+here = path.abspath(path.dirname(__file__))
+
+# Get the long description from the README file
+with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+ long_description = f.read()
+
def parse_requirements(name=None):
if name:
reqf = 'requirements-%s.txt' % name
else:
reqf = 'requirements.txt'
requirements = []
- if not os.path.exists(reqf):
+ if not path.exists(reqf):
return requirements
with open(reqf) as f:
for line in f.readlines():
line = line.strip()
if not line or line.startswith('#'):
continue
requirements.append(line)
return requirements
setup(
name='swh.loader.pypi',
description='Software Heritage PyPI Loader',
+ long_description=long_description,
+ long_description_content_type='text/markdown',
author='Software Heritage developers',
author_email='swh-devel@inria.fr',
- url='https://forge.softwareheritage.org/source/swh-loader-pypi.git',
+ url='https://forge.softwareheritage.org/source/swh-loader-pypi',
packages=find_packages(),
scripts=[], # scripts to package
install_requires=parse_requirements() + parse_requirements('swh'),
test_requires=parse_requirements('test'),
setup_requires=['vcversioner'],
+ extras_require={'testing': parse_requirements('test')},
vcversioner={'version_module_paths': ['swh/loader/pypi/_version.py']},
include_package_data=True,
+ classifiers=[
+ "Programming Language :: Python :: 3",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+ "Operating System :: OS Independent",
+ "Development Status :: 5 - Production/Stable",
+ ],
+ project_urls={
+ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest',
+ 'Funding': 'https://www.softwareheritage.org/donate',
+ 'Source': 'https://forge.softwareheritage.org/source/swh-loader-pypi',
+ },
)
diff --git a/swh.loader.pypi.egg-info/PKG-INFO b/swh.loader.pypi.egg-info/PKG-INFO
index e9f3971..b04d0da 100644
--- a/swh.loader.pypi.egg-info/PKG-INFO
+++ b/swh.loader.pypi.egg-info/PKG-INFO
@@ -1,10 +1,136 @@
-Metadata-Version: 1.0
+Metadata-Version: 2.1
Name: swh.loader.pypi
-Version: 0.0.2
+Version: 0.0.3
Summary: Software Heritage PyPI Loader
-Home-page: https://forge.softwareheritage.org/source/swh-loader-pypi.git
+Home-page: https://forge.softwareheritage.org/source/swh-loader-pypi
Author: Software Heritage developers
Author-email: swh-devel@inria.fr
License: UNKNOWN
-Description: UNKNOWN
+Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
+Project-URL: Funding, https://www.softwareheritage.org/donate
+Project-URL: Source, https://forge.softwareheritage.org/source/swh-loader-pypi
+Description: swh-loader-pypi
+ ====================
+
+ SWH PyPI loader's source code repository
+
+ # What does the loader do?
+
+ The PyPI loader visits and loads a PyPI project [1].
+
+ Each visit will result in:
+ - 1 snapshot (which targets n revisions ; 1 per release artifact)
+ - 1 revision (which targets 1 directory ; the release artifact uncompressed)
+
+ [1] https://pypi.org/help/#packages
+
+ ## First visit
+
+ Given a PyPI project (origin), the loader, for the first visit:
+
+ - retrieves information for the given project (including releases)
+ - then for each associated release
+ - for each associated source distribution (type 'sdist') release
+ artifact (possibly many per release)
+ - retrieves the associated artifact archive (with checks)
+ - uncompresses locally the archive
+ - computes the hashes of the uncompressed directory
+ - then creates a revision (using PKG-INFO metadata file) targeting
+ such directory
+ - finally, creates a snapshot targeting all seen revisions
+ (uncompressed PyPI artifact and metadata).
+
+ ## Next visit
+
+ The loader starts by checking if something changed since the last
+ visit. If nothing changed, the visit's snapshot is left
+ unchanged. The new visit targets the same snapshot.
+
+ If something changed, the already seen release artifacts are skipped.
+ Only the new ones are loaded. In the end, the loader creates a new
+ snapshot based on the previous one. Thus, the new snapshot targets
+ both the old and new PyPI release artifacts.
+
+ ## Terminology
+
+ - 1 project: a PyPI project (used as swh origin). This is a collection
+ of releases.
+
+ - 1 release: a specific version of the (PyPi) project. It's a
+ collection of information and associated source release
+ artifacts (type 'sdist')
+
+ - 1 release artifact: a source release artifact (distributed by a PyPI
+ maintainer). In swh, we are specifically
+ interested by the 'sdist' type (source code).
+
+ ## Edge cases
+
+ - If no release provides release artifacts, those are skipped
+
+ - If a release artifact holds no PKG-INFO file (root at the archive),
+ the release artifact is skipped.
+
+ - If a problem occurs during a fetch action (e.g. release artifact
+ download), the load fails and the visit is marked as 'partial'.
+
+ # Development
+
+ ## Configuration file
+
+ ### Location
+
+ Either:
+ - /etc/softwareheritage/
+ - ~/.config/swh/
+ - ~/.swh/
+
+ Note: Will call that location $SWH_CONFIG_PATH
+
+ ### Configuration sample
+
+ $SWH_CONFIG_PATH/loader/pypi.yml:
+ ```
+ storage:
+ cls: remote
+ args:
+ url: http://localhost:5002/
+
+ ```
+
+ ## Local run
+
+ The built-in command-line will run the loader for a project in the
+ main PyPI archive.
+
+ For instance, to load arrow:
+ ``` sh
+ python3 -m swh.loader.pypi.loader arrow
+ ```
+
+ If you need more control, you can use the loader directly. It expects
+ three arguments:
+ - project: a PyPI project name (f.e.: arrow)
+ - project_url: URL of the PyPI project (human-readable html page)
+ - project_metadata_url: URL of the PyPI metadata information
+ (machine-parsable json document)
+
+ ``` python
+ import logging
+ logging.basicConfig(level=logging.DEBUG)
+
+ from swh.loader.pypi.tasks import LoadPyPI
+
+ project='arrow'
+
+ LoadPyPI().run(project, 'https://pypi.org/pypi/%s/' % project, 'https://pypi.org/pypi/%s/json' % project)
+ ```
+
Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Description-Content-Type: text/markdown
+Provides-Extra: testing
diff --git a/swh.loader.pypi.egg-info/SOURCES.txt b/swh.loader.pypi.egg-info/SOURCES.txt
index 84983df..36ce350 100644
--- a/swh.loader.pypi.egg-info/SOURCES.txt
+++ b/swh.loader.pypi.egg-info/SOURCES.txt
@@ -1,50 +1,49 @@
.gitignore
AUTHORS
LICENSE
MANIFEST.in
Makefile
README.md
requirements-swh.txt
requirements-test.txt
requirements.txt
setup.py
version.txt
debian/changelog
debian/compat
debian/control
debian/copyright
debian/rules
debian/source/format
docs/.gitignore
docs/Makefile
docs/conf.py
docs/index.rst
docs/_static/.placeholder
docs/_templates/.placeholder
swh/__init__.py
swh.loader.pypi.egg-info/PKG-INFO
swh.loader.pypi.egg-info/SOURCES.txt
swh.loader.pypi.egg-info/dependency_links.txt
swh.loader.pypi.egg-info/requires.txt
swh.loader.pypi.egg-info/top_level.txt
swh/loader/__init__.py
swh/loader/pypi/.gitignore
swh/loader/pypi/__init__.py
swh/loader/pypi/_version.py
swh/loader/pypi/client.py
swh/loader/pypi/converters.py
swh/loader/pypi/loader.py
-swh/loader/pypi/model.py
swh/loader/pypi/tasks.py
swh/loader/pypi/tests/__init__.py
swh/loader/pypi/tests/common.py
swh/loader/pypi/tests/test_client.py
swh/loader/pypi/tests/test_converters.py
swh/loader/pypi/tests/test_loader.py
swh/loader/pypi/tests/resources/0805nexter+new-made-up-release.json
swh/loader/pypi/tests/resources/0805nexter-unpublished-release.json
swh/loader/pypi/tests/resources/0805nexter.json
swh/loader/pypi/tests/resources/archives/0805nexter-1.1.0.zip
swh/loader/pypi/tests/resources/archives/0805nexter-1.2.0.zip
swh/loader/pypi/tests/resources/archives/0805nexter-1.3.0.zip
swh/loader/pypi/tests/resources/archives/0805nexter-1.4.0.zip
\ No newline at end of file
diff --git a/swh.loader.pypi.egg-info/requires.txt b/swh.loader.pypi.egg-info/requires.txt
index 4abe634..78f4d33 100644
--- a/swh.loader.pypi.egg-info/requires.txt
+++ b/swh.loader.pypi.egg-info/requires.txt
@@ -1,10 +1,13 @@
arrow
pkginfo
requests
setuptools
swh.core
-swh.loader.core
+swh.loader.core>=0.0.34
swh.model>=0.0.27
swh.scheduler
swh.storage
vcversioner
+
+[testing]
+nose
diff --git a/swh/loader/pypi/_version.py b/swh/loader/pypi/_version.py
index f222511..4fee5ff 100644
--- a/swh/loader/pypi/_version.py
+++ b/swh/loader/pypi/_version.py
@@ -1,5 +1,5 @@
# This file is automatically generated by setup.py.
-__version__ = '0.0.2'
-__sha__ = 'gf140bd0'
-__revision__ = 'gf140bd0'
+__version__ = '0.0.3'
+__sha__ = 'gb237da9'
+__revision__ = 'gb237da9'
diff --git a/swh/loader/pypi/client.py b/swh/loader/pypi/client.py
index 1247930..73ae266 100644
--- a/swh/loader/pypi/client.py
+++ b/swh/loader/pypi/client.py
@@ -1,463 +1,463 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from collections import defaultdict
import logging
import os
import shutil
import arrow
from pkginfo import UnpackedSDist
import requests
from swh.core import tarball
from swh.model import hashutil
from .converters import info, author
try:
from swh.loader.pypi._version import __version__
except ImportError:
__version__ = 'devel'
def _to_dict(pkginfo):
"""Given a pkginfo parsed structure, convert it to a dict.
Args:
pkginfo (UnpackedSDist): The sdist parsed structure
Returns:
parsed structure as a dict
"""
m = {}
for k in pkginfo:
m[k] = getattr(pkginfo, k)
return m
def _project_pkginfo(dir_path):
"""Given an uncompressed path holding the pkginfo file, returns a
pkginfo parsed structure as a dict.
The release artifact contains at their root one folder. For example:
$ tar tvf zprint-0.0.6.tar.gz
drwxr-xr-x root/root 0 2018-08-22 11:01 zprint-0.0.6/
...
Args:
dir_path (str): Path to the uncompressed directory
representing a release artifact from pypi.
Returns:
the pkginfo parsed structure as a dict if any or None if
none was present.
"""
# Retrieve the root folder of the archive
project_dirname = os.listdir(dir_path)[0]
pkginfo_path = os.path.join(dir_path, project_dirname, 'PKG-INFO')
if not os.path.exists(pkginfo_path):
return None
pkginfo = UnpackedSDist(pkginfo_path)
return _to_dict(pkginfo)
class PyPIClient:
"""PyPI client in charge of discussing with the pypi server.
Args:
base_url (str): PyPI instance's base url
temp_directory (str): Path to the temporary disk location used
for uncompressing the release artifacts
cache (bool): Use an internal cache to keep the archives on
disk. Default is not to use it.
cache_dir (str): cache's disk location (relevant only with
`cache` to True)
Those last 2 parameters are not for production use.
"""
def __init__(self, base_url='https://pypi.org/pypi',
temp_directory=None, cache=False, cache_dir=None):
self.version = __version__
self.base_url = base_url
self.temp_directory = temp_directory
self.do_cache = cache
if self.do_cache:
self.cache_dir = cache_dir
self.cache_raw_dir = os.path.join(cache_dir, 'archives')
os.makedirs(self.cache_raw_dir, exist_ok=True)
self.session = requests.session()
self.params = {
'headers': {
'User-Agent': 'Software Heritage PyPI Loader (%s)' % (
__version__
)
}
}
def _save_response(self, response, project=None):
"""Log the response from a server request to a cache dir.
Args:
response (Response): full server response
cache_dir (str): system path for cache dir
Returns:
nothing
"""
import gzip
from json import dumps
datepath = arrow.utcnow().isoformat()
name = '%s.gz' % datepath if project is None else '%s-%s.gz' % (
project, datepath)
fname = os.path.join(self.cache_dir, name)
with gzip.open(fname, 'w') as f:
f.write(bytes(
dumps(response.json()),
'utf-8'
))
def _save_raw(self, filepath):
"""In cache mode, backup the filepath to self.cache_raw_dir
Args:
filepath (str): Path of the file to save
"""
_filename = os.path.basename(filepath)
_archive = os.path.join(self.cache_raw_dir, _filename)
shutil.copyfile(filepath, _archive)
def _get_raw(self, filepath):
"""In cache mode, we try to retrieve the cached file.
"""
_filename = os.path.basename(filepath)
_archive = os.path.join(self.cache_raw_dir, _filename)
if not os.path.exists(_archive):
return None
shutil.copyfile(_archive, filepath)
return filepath
def _get(self, url, project=None):
"""Get query to the url.
Args:
url (str): Url
Raises:
ValueError in case of failing to query
Returns:
Response as dict if ok
"""
response = self.session.get(url, **self.params)
if response.status_code != 200:
raise ValueError("Fail to query '%s'. Reason: %s" % (
url, response.status_code))
if self.do_cache:
self._save_response(response, project=project)
return response.json()
def info(self, project_url, project=None):
"""Given a metadata project url, retrieve the raw json response
Args:
project_url (str): Project's pypi to retrieve information
Returns:
Main project information as dict.
"""
return self._get(project_url, project=project)
def release(self, project, release):
"""Given a project and a release name, retrieve the raw information
for said project's release.
Args:
project (str): Project's name
release (dict): Release information
Returns:
Release information as dict
"""
release_url = '%s/%s/%s/json' % (self.base_url, project, release)
return self._get(release_url, project=project)
def prepare_release_artifacts(self, project, version, release_artifacts):
"""For a given project's release version, fetch and prepare the
associated release artifacts.
Args:
project (str): PyPI Project
version (str): Release version
release_artifacts ([dict]): List of source distribution
release artifacts
Yields:
tuple (artifact, filepath, uncompressed_path, pkginfo) where:
- artifact (dict): release artifact's associated info
- release (dict): release information
- filepath (str): Local artifact's path
- uncompressed_archive_path (str): uncompressed archive path
- pkginfo (dict): package information or None if none found
"""
for artifact in release_artifacts:
release = {
'name': version,
'message': artifact.get('comment_text', ''),
}
artifact = {
'sha256': artifact['digests']['sha256'],
'size': artifact['size'],
'filename': artifact['filename'],
'url': artifact['url'],
'date': artifact['upload_time'],
}
yield self.prepare_release_artifact(project, release, artifact)
def prepare_release_artifact(self, project, release, artifact):
"""For a given release project, fetch and prepare the associated
artifact.
This:
- fetches the artifact
- checks the size, hashes match
- uncompress the artifact locally
- computes the swh hashes
- returns the associated information for the artifact
Args:
project (str): Project's name
release (dict): Release information
artifact (dict): Release artifact information
Returns:
tuple (artifact, filepath, uncompressed_path, pkginfo) where:
- release (dict): Release information (name, message)
- artifact (dict): release artifact's information
- filepath (str): Local artifact's path
- uncompressed_archive_path (str): uncompressed archive path
- pkginfo (dict): package information or None if none found
"""
version = release['name']
logging.debug('Release version: %s' % version)
path = os.path.join(self.temp_directory, project, version)
os.makedirs(path, exist_ok=True)
filepath = os.path.join(path, artifact['filename'])
logging.debug('Artifact local path: %s' % filepath)
cache_hit = None
if self.do_cache:
cache_hit = self._get_raw(filepath)
if cache_hit:
- h = hashutil.MultiHash.from_path(filepath, track_length=False)
+ h = hashutil.MultiHash.from_path(filepath)
else: # no cache hit, we fetch from pypi
url = artifact['url']
r = self.session.get(url, **self.params, stream=True)
status = r.status_code
if status != 200:
if status == 404:
raise ValueError("Project '%s' not found" % url)
else:
msg = "Fail to query '%s'\nCode: %s\nDetails: %s" % (
url, r.status_code, r.content)
raise ValueError(msg)
length = int(r.headers['content-length'])
if length != artifact['size']:
raise ValueError('Error when checking size: %s != %s' % (
artifact['size'], length))
h = hashutil.MultiHash(length=length)
with open(filepath, 'wb') as f:
for chunk in r.iter_content():
h.update(chunk)
f.write(chunk)
hashes = h.hexdigest()
actual_digest = hashes['sha256']
if actual_digest != artifact['sha256']:
raise ValueError(
'%s %s: Checksum mismatched: %s != %s' % (
project, version, artifact['sha256'], actual_digest))
if not cache_hit and self.do_cache:
self._save_raw(filepath)
uncompress_path = os.path.join(path, 'uncompress')
os.makedirs(uncompress_path, exist_ok=True)
nature = tarball.uncompress(filepath, uncompress_path)
artifact['archive_type'] = nature
artifact.update(hashes)
pkginfo = _project_pkginfo(uncompress_path)
return release, artifact, filepath, uncompress_path, pkginfo
class PyPIProject:
"""PyPI project representation
This allows to extract information for a given project:
- either its latest information (from the latest release)
- either for a given release version
- uncompress associated fetched release artifacts
This also fetches and uncompresses the associated release
artifacts.
"""
def __init__(self, client, project, project_metadata_url, data=None):
self.client = client
self.project = project
self.project_metadata_url = project_metadata_url
if data:
self.data = data
else:
self.data = client.info(project_metadata_url, project)
self.last_version = self.data['info']['version']
self.cache = {
self.last_version: self.data
}
def _data(self, release_name=None):
"""Fetch data per release and cache it. Returns the cache retrieved
data if already fetched.
"""
if release_name:
data = self.cache.get(release_name)
if not data:
data = self.client.release(self.project, release_name)
self.cache[release_name] = data
else:
data = self.data
return data
def info(self, release_name=None):
"""Compute release information for provided release (or latest one).
"""
return info(self._data(release_name))
def _filter_release_artifacts(self, version, releases,
known_artifacts=None):
"""Filter not already known sdist (source distribution) release.
There can be multiple 'package_type' (sdist, bdist_egg,
bdist_wheel, bdist_rpm, bdist_msi, bdist_wininst, ...), we are
only interested in source distribution (sdist), others bdist*
are binary
Args:
version (str): Release name or version
releases (dict/[dict]): Full release object (or a list of)
known_artifacts ([tuple]): List of known releases (tuple filename,
sha256)
Yields:
an unknown release artifact
"""
if not releases:
return []
if not isinstance(releases, list):
releases = [releases]
if not known_artifacts:
known_artifacts = set()
for artifact in releases:
name = artifact['filename']
sha256 = artifact['digests']['sha256']
if (name, sha256) in known_artifacts:
logging.debug('artifact (%s, %s) already seen for release %s, skipping' % ( # noqa
name, sha256, version))
continue
if artifact['packagetype'] != 'sdist':
continue
yield artifact
def _cleanup_release_artifacts(self, archive_path, directory_path):
"""Clean intermediary files which no longer needs to be present.
"""
if directory_path and os.path.exists(directory_path):
logging.debug('Clean up uncompressed archive path %s' % (
directory_path, ))
shutil.rmtree(directory_path)
if archive_path and os.path.exists(archive_path):
logging.debug('Clean up archive %s' % archive_path)
os.unlink(archive_path)
def all_release_artifacts(self):
"""Generate a mapping of releases to their artifacts"""
ret = defaultdict(list)
for version, artifacts in self.data['releases'].items():
for artifact in self._filter_release_artifacts(version, artifacts):
ret[version].append((artifact['filename'],
artifact['digests']['sha256']))
return ret
def default_release(self):
"""Return the version number of the default release,
as would be installed by `pip install`"""
return self.data['info']['version']
def download_new_releases(self, known_artifacts):
"""Fetch metadata/data per release (if new release artifact detected)
For new release artifact, this:
- downloads and uncompresses the release artifacts.
- yields the (release info, author info, release, dir_path)
- Clean up the intermediary fetched artifact files
Args:
known_artifacts (tuple): artifact name, artifact sha256 hash
Yields:
tuple (version, release_info, release, uncompressed_path) where:
- project_info (dict): release's associated version info
- author (dict): Author information for the release
- artifact (dict): Release artifact information
- release (dict): release metadata
- uncompressed_path (str): Path to uncompressed artifact
"""
releases_dict = self.data['releases']
for version, releases in releases_dict.items():
releases = self._filter_release_artifacts(
version, releases, known_artifacts)
releases = self.client.prepare_release_artifacts(
self.project, version, releases)
for release, artifact, archive, dir_path, pkginfo in releases:
if pkginfo is None: # fallback to pypi api metadata
msg = '%s %s: No PKG-INFO detected, skipping' % ( # noqa
self.project, version)
logging.warn(msg)
continue
yield pkginfo, author(pkginfo), release, artifact, dir_path
self._cleanup_release_artifacts(archive, dir_path)
diff --git a/swh/loader/pypi/loader.py b/swh/loader/pypi/loader.py
index 98e6b66..32f5b53 100644
--- a/swh/loader/pypi/loader.py
+++ b/swh/loader/pypi/loader.py
@@ -1,307 +1,307 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
import shutil
from tempfile import mkdtemp
import arrow
from swh.loader.core.utils import clean_dangling_folders
from swh.loader.core.loader import SWHLoader
from swh.model.from_disk import Directory
from swh.model.identifiers import (
revision_identifier, snapshot_identifier,
identifier_to_bytes, normalize_timestamp
)
from .client import PyPIClient, PyPIProject
TEMPORARY_DIR_PREFIX_PATTERN = 'swh.loader.pypi.'
DEBUG_MODE = '** DEBUG MODE **'
class PyPILoader(SWHLoader):
CONFIG_BASE_FILENAME = 'loader/pypi'
ADDITIONAL_CONFIG = {
'temp_directory': ('str', '/tmp/swh.loader.pypi/'),
'cache': ('bool', False),
'cache_dir': ('str', ''),
'debug': ('bool', False), # NOT FOR PRODUCTION
}
def __init__(self, client=None):
super().__init__(logging_class='swh.loader.pypi.PyPILoader')
self.origin_id = None
if not client:
temp_directory = self.config['temp_directory']
os.makedirs(temp_directory, exist_ok=True)
self.temp_directory = mkdtemp(
suffix='-%s' % os.getpid(),
prefix=TEMPORARY_DIR_PREFIX_PATTERN,
dir=temp_directory)
self.pypi_client = PyPIClient(
temp_directory=self.temp_directory,
cache=self.config['cache'],
cache_dir=self.config['cache_dir'])
else:
self.temp_directory = client.temp_directory
self.pypi_client = client
self.debug = self.config['debug']
self.done = False
def pre_cleanup(self):
"""To prevent disk explosion if some other workers exploded
in mid-air (OOM killed), we try and clean up dangling files.
"""
if self.debug:
self.log.warn('%s Will not pre-clean up temp dir %s' % (
DEBUG_MODE, self.temp_directory
))
return
clean_dangling_folders(self.config['temp_directory'],
pattern_check=TEMPORARY_DIR_PREFIX_PATTERN,
log=self.log)
def cleanup(self):
"""Clean up temporary disk use
"""
if self.debug:
self.log.warn('%s Will not clean up temp dir %s' % (
DEBUG_MODE, self.temp_directory
))
return
if os.path.exists(self.temp_directory):
self.log.debug('Clean up %s' % self.temp_directory)
shutil.rmtree(self.temp_directory)
- def prepare_origin_visit(self, project_name, origin_url,
- origin_metadata_url=None):
+ def prepare_origin_visit(self, project_name, project_url,
+ project_metadata_url=None):
"""Prepare the origin visit information
Args:
project_name (str): Project's simple name
- origin_url (str): Project's main url
- origin_metadata_url (str): Project's metadata url
+ project_url (str): Project's main url
+ project_metadata_url (str): Project's metadata url
"""
self.origin = {
- 'url': origin_url,
+ 'url': project_url,
'type': 'pypi',
}
self.visit_date = None # loader core will populate it
def _known_artifacts(self, last_snapshot):
"""Retrieve the known releases/artifact for the origin_id.
Args
snapshot (dict): Last snapshot for the visit
Returns:
list of (filename, sha256) tuples.
"""
if not last_snapshot or 'branches' not in last_snapshot:
return {}
revs = [rev['target'] for rev in last_snapshot['branches'].values()]
known_revisions = self.storage.revision_get(revs)
ret = {}
for revision in known_revisions:
if 'original_artifact' in revision['metadata']:
artifact = revision['metadata']['original_artifact']
ret[artifact['filename'], artifact['sha256']] = revision['id']
return ret
def _last_snapshot(self):
"""Retrieve the last snapshot
"""
return self.storage.snapshot_get_latest(self.origin_id)
- def prepare(self, project_name, origin_url,
- origin_metadata_url=None):
+ def prepare(self, project_name, project_url,
+ project_metadata_url=None):
"""Keep reference to the origin url (project) and the
project metadata url
Args:
project_name (str): Project's simple name
- origin_url (str): Project's main url
- origin_metadata_url (str): Project's metadata url
+ project_url (str): Project's main url
+ project_metadata_url (str): Project's metadata url
"""
self.project_name = project_name
- self.origin_url = origin_url
- self.origin_metadata_url = origin_metadata_url
+ self.origin_url = project_url
+ self.project_metadata_url = project_metadata_url
self.project = PyPIProject(self.pypi_client, self.project_name,
- self.origin_metadata_url)
+ self.project_metadata_url)
self._prepare_state()
def _prepare_state(self):
"""Initialize internal state (snapshot, contents, directories, etc...)
This is called from `prepare` method.
"""
last_snapshot = self._last_snapshot()
self.known_artifacts = self._known_artifacts(last_snapshot)
# and the artifacts
# that will be the source of data to retrieve
self.new_artifacts = self.project.download_new_releases(
self.known_artifacts
)
# temporary state
self._contents = []
self._directories = []
self._revisions = []
self._load_status = 'uneventful'
self._visit_status = 'full'
def fetch_data(self):
"""Called once per release artifact version (can be many for one
release).
This will for each call:
- retrieve a release artifact (associated to a release version)
- Uncompress it and compute the necessary information
- Computes the swh objects
Returns:
True as long as data to fetch exist
"""
data = None
if self.done:
return False
try:
data = next(self.new_artifacts)
self._load_status = 'eventful'
except StopIteration:
self.done = True
return False
project_info, author, release, artifact, dir_path = data
dir_path = dir_path.encode('utf-8')
directory = Directory.from_disk(path=dir_path, data=True)
_objects = directory.collect()
self._contents = _objects['content'].values()
self._directories = _objects['directory'].values()
date = normalize_timestamp(
int(arrow.get(artifact['date']).timestamp))
name = release['name'].encode('utf-8')
message = release['message'].encode('utf-8')
if message:
message = b'%s: %s' % (name, message)
else:
message = name
_revision = {
'synthetic': True,
'metadata': {
'original_artifact': artifact,
'project': project_info,
},
'author': author,
'date': date,
'committer': author,
'committer_date': date,
'message': message,
'directory': directory.hash,
'parents': [],
'type': 'tar',
}
_revision['id'] = identifier_to_bytes(
revision_identifier(_revision))
self._revisions.append(_revision)
artifact_key = artifact['filename'], artifact['sha256']
self.known_artifacts[artifact_key] = _revision['id']
return not self.done
def target_from_artifact(self, filename, sha256):
target = self.known_artifacts.get((filename, sha256))
if target:
return {
'target': target,
'target_type': 'revision',
}
return None
def generate_and_load_snapshot(self):
branches = {}
for release, artifacts in self.project.all_release_artifacts().items():
default_release = self.project.default_release()
if len(artifacts) == 1:
# Only one artifact for this release, generate a single branch
branch_name = 'releases/%s' % release
filename, sha256 = artifacts[0]
target = self.target_from_artifact(filename, sha256)
branches[branch_name.encode('utf-8')] = target
if release == default_release:
branches[b'HEAD'] = {
'target_type': 'alias',
'target': branch_name.encode('utf-8'),
}
if not target:
self._visit_status = 'partial'
else:
# Several artifacts for this release, generate a separate
# pointer for each of them
for filename, sha256 in artifacts:
branch_name = 'releases/%s/%s' % (release, filename)
target = self.target_from_artifact(filename, sha256)
branches[branch_name.encode('utf-8')] = target
if not target:
self._visit_status = 'partial'
snapshot = {
'branches': branches,
}
snapshot['id'] = identifier_to_bytes(
snapshot_identifier(snapshot))
self.maybe_load_snapshot(snapshot)
def store_data(self):
"""(override) This sends collected objects to storage.
"""
self.maybe_load_contents(self._contents)
self.maybe_load_directories(self._directories)
self.maybe_load_revisions(self._revisions)
if self.done:
self.generate_and_load_snapshot()
self.flush()
def load_status(self):
return {
'status': self._load_status,
}
def visit_status(self):
return self._visit_status
if __name__ == '__main__':
import logging
import sys
logging.basicConfig(level=logging.DEBUG)
if len(sys.argv) != 2:
logging.error('Usage: %s ' % sys.argv[0])
sys.exit(1)
module_name = sys.argv[1]
loader = PyPILoader()
loader.load(
module_name,
'https://pypi.org/projects/%s/' % module_name,
'https://pypi.org/pypi/%s/json' % module_name,
)
diff --git a/swh/loader/pypi/model.py b/swh/loader/pypi/model.py
deleted file mode 100644
index d823892..0000000
--- a/swh/loader/pypi/model.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright (C) 2018 The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
-
-import os
-import logging
-import shutil
-
-from .converters import info, author
diff --git a/swh/loader/pypi/tasks.py b/swh/loader/pypi/tasks.py
index b915cde..51e4878 100644
--- a/swh/loader/pypi/tasks.py
+++ b/swh/loader/pypi/tasks.py
@@ -1,19 +1,19 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.scheduler.task import Task
from .loader import PyPILoader
class LoadPyPI(Task):
task_queue = 'swh_loader_pypi'
def run_task(self, project_name, project_url, project_metadata_url=None):
loader = PyPILoader()
loader.log = self.log
return loader.load(project_name,
project_url,
- origin_metadata_url=project_metadata_url)
+ project_metadata_url=project_metadata_url)
diff --git a/swh/loader/pypi/tests/common.py b/swh/loader/pypi/tests/common.py
index 0d557a7..8acbea0 100644
--- a/swh/loader/pypi/tests/common.py
+++ b/swh/loader/pypi/tests/common.py
@@ -1,151 +1,56 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
import shutil
import os
import tempfile
from nose.plugins.attrib import attr
from unittest import TestCase
from swh.loader.pypi.client import PyPIClient, PyPIProject
RESOURCES_PATH = './swh/loader/pypi/tests/resources'
class PyPIClientWithCache(PyPIClient):
"""Force the use of the cache to bypass pypi calls
"""
def __init__(self, temp_directory, cache_dir):
super().__init__(temp_directory=temp_directory,
cache=True, cache_dir=cache_dir)
-class LoaderNoStorage:
- """Mixin class to inhibit the persistence (storage calls) and keep in
- memory the data sent.
-
- """
- CONFIG_BASE_FILENAME = '' # do not provide a real path
- ADDITIONAL_CONFIG = {
- 'storage': ('dict', {
- 'cls': 'remote',
- 'args': {
- 'url': 'http://nowhere:5002/', # do not provide a real storage
- }
- }),
-
- # do not send any data to the storage
- 'send_contents': ('bool', False),
- 'send_directories': ('bool', False),
- 'send_revisions': ('bool', False),
- 'send_releases': ('bool', False),
- 'send_snapshot': ('bool', False),
- 'debug': ('bool', False),
- }
-
- def __init__(self, client=None):
- super().__init__(client=client)
- self.all_contents = []
- self.all_directories = []
- self.all_revisions = []
- self.all_releases = []
- self.all_snapshots = []
-
- # typed data
- self.objects = {
- 'content': self.all_contents,
- 'directory': self.all_directories,
- 'revision': self.all_revisions,
- 'release': self.all_releases,
- 'snapshot': self.all_snapshots
- }
-
- def _add(self, type, l):
- """Add without duplicates and keeping the insertion order.
-
- Args:
- type (str): Type of objects concerned by the action
- l ([object]): List of 'type' object
-
- """
- col = self.objects[type]
- for o in l:
- if o in col:
- continue
- col.extend([o])
-
- def maybe_load_contents(self, all_contents):
- self._add('content', all_contents)
-
- def maybe_load_directories(self, all_directories):
- self._add('directory', all_directories)
-
- def maybe_load_revisions(self, all_revisions):
- self._add('revision', all_revisions)
-
- def maybe_load_releases(self, releases):
- raise ValueError('If called, the test must break.')
-
- def maybe_load_snapshot(self, snapshot):
- self.objects['snapshot'].append(snapshot)
-
- def _store_origin_visit(self):
- pass
-
- def open_fetch_history(self):
- pass
-
- def close_fetch_history_success(self, fetch_history_id):
- pass
-
- def close_fetch_history_failure(self, fetch_history_id):
- pass
-
- def update_origin_visit(self, origin_id, visit, status):
- pass
-
- # Override to do nothing at the end
- def close_failure(self):
- pass
-
- def close_success(self):
- pass
-
- def pre_cleanup(self):
- pass
-
-
@attr('fs')
class WithProjectTest(TestCase):
def setUp(self):
project = '0805nexter'
project_metadata_file = '%s/%s.json' % (RESOURCES_PATH, project)
with open(project_metadata_file) as f:
data = json.load(f)
temp_dir = tempfile.mkdtemp(
dir='/tmp/', prefix='swh.loader.pypi.tests-')
project_metadata_url = 'https://pypi.org/pypi/%s/json' % project
# Will use the pypi with cache
client = PyPIClientWithCache(
temp_directory=temp_dir, cache_dir=RESOURCES_PATH)
self.project = PyPIProject(
client=client,
project=project,
project_metadata_url=project_metadata_url,
data=data)
self.data = data
self.temp_dir = temp_dir
self.project_name = project
def tearDown(self):
if os.path.exists(self.temp_dir):
shutil.rmtree(self.temp_dir)
diff --git a/swh/loader/pypi/tests/test_client.py b/swh/loader/pypi/tests/test_client.py
index be10392..f364b61 100644
--- a/swh/loader/pypi/tests/test_client.py
+++ b/swh/loader/pypi/tests/test_client.py
@@ -1,102 +1,97 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
-from nose.tools import istest
-
from swh.loader.pypi import converters
from swh.loader.pypi.client import _project_pkginfo
from .common import WithProjectTest
class PyPIProjectTest(WithProjectTest):
- @istest
- def download_new_releases(self):
+ def test_download_new_releases(self):
actual_releases = self.project.download_new_releases([])
expected_release_artifacts = {
'1.1.0': {
'archive_type': 'zip',
'blake2s256': 'df9413bde66e6133b10cadefad6fcf9cbbc369b47831089112c846d79f14985a', # noqa
'date': '2016-01-31T05:28:42',
'filename': '0805nexter-1.1.0.zip',
'sha1': '127d8697db916ba1c67084052196a83319a25000',
'sha1_git': '4b8f1350e6d9fa00256e974ae24c09543d85b196',
'sha256': '52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035', # noqa
'size': 862,
'url': 'https://files.pythonhosted.org/packages/ec/65/c0116953c9a3f47de89e71964d6c7b0c783b01f29fa3390584dbf3046b4d/0805nexter-1.1.0.zip', # noqa
},
'1.2.0': {
'archive_type': 'zip',
'blake2s256': '67010586b5b9a4aaa3b1c386f9dc8b4c99e6e40f37732a717a5f9b9b1185e588', # noqa
'date': '2016-01-31T05:51:25',
'filename': '0805nexter-1.2.0.zip',
'sha1': 'd55238554b94da7c5bf4a349ece0fe3b2b19f79c',
'sha1_git': '8638d33a96cb25d8319af21417f00045ec6ee810',
'sha256': '49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709', # noqa
'size': 898,
'url': 'https://files.pythonhosted.org/packages/c4/a0/4562cda161dc4ecbbe9e2a11eb365400c0461845c5be70d73869786809c4/0805nexter-1.2.0.zip', # noqa
}
}
expected_releases = {
'1.1.0': {
'name': '1.1.0',
'message': '',
},
'1.2.0': {
'name': '1.2.0',
'message': '',
},
}
dir_paths = []
for pkginfo, author, release, artifact, dir_path in actual_releases:
version = pkginfo['version']
expected_pkginfo = _project_pkginfo(dir_path)
self.assertEquals(pkginfo, expected_pkginfo)
expected_author = converters.author(expected_pkginfo)
self.assertEqual(author, expected_author)
expected_artifact = expected_release_artifacts[version]
self.assertEqual(artifact, expected_artifact)
expected_release = expected_releases[version]
self.assertEqual(release, expected_release)
self.assertTrue(version in dir_path)
self.assertTrue(self.project_name in dir_path)
# path still exists
self.assertTrue(os.path.exists(dir_path))
dir_paths.append(dir_path)
# Ensure uncompressed paths have been destroyed
for dir_path in dir_paths:
# path no longer exists
self.assertFalse(os.path.exists(dir_path))
- @istest
- def all_release_artifacts(self):
+ def test_all_release_artifacts(self):
expected_release_artifacts = {
'1.1.0': [(
'0805nexter-1.1.0.zip',
'52cd128ad3afe539478abc7440d4b043'
'384295fbe6b0958a237cb6d926465035',
)],
'1.2.0': [(
'0805nexter-1.2.0.zip',
'49785c6ae39ea511b3c253d7621c0b1b'
'6228be2f965aca8a491e6b84126d0709',
)],
}
self.assertEqual(
self.project.all_release_artifacts(),
expected_release_artifacts,
)
- @istest
- def default_release(self):
+ def test_default_release(self):
self.assertEqual(self.project.default_release(), '1.2.0')
diff --git a/swh/loader/pypi/tests/test_converters.py b/swh/loader/pypi/tests/test_converters.py
index 0e2f804..effca39 100644
--- a/swh/loader/pypi/tests/test_converters.py
+++ b/swh/loader/pypi/tests/test_converters.py
@@ -1,130 +1,121 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from unittest import TestCase
-from nose.tools import istest
-from swh.loader.pypi.converters import author, EMPTY_AUTHOR
+from swh.loader.pypi.converters import EMPTY_AUTHOR, author
from .common import WithProjectTest
class Test(WithProjectTest):
- @istest
- def info(self):
+ def test_info(self):
actual_info = self.project.info()
expected_info = {
'home_page': self.data['info']['home_page'],
'description': self.data['info']['description'],
'summary': self.data['info']['summary'],
'license': self.data['info']['license'],
'package_url': self.data['info']['package_url'],
'project_url': self.data['info']['project_url'],
'upstream': self.data['info']['project_urls']['Homepage'],
}
self.assertEqual(expected_info, actual_info)
- @istest
- def author(self):
+ def test_author(self):
info = self.data['info']
actual_author = author(info)
name = info['author'].encode('utf-8')
email = info['author_email'].encode('utf-8')
expected_author = {
'fullname': b'%s <%s>' % (name, email),
'name': name,
'email': email,
}
self.assertEqual(expected_author, actual_author)
- @istest
- def no_author(self):
+ def test_no_author(self):
actual_author = author({})
self.assertEqual(EMPTY_AUTHOR, actual_author)
- @istest
- def partial_author(self):
+ def test_partial_author(self):
actual_author = author({'author': 'someone'})
expected_author = {
'name': b'someone',
'fullname': b'someone',
'email': None,
}
self.assertEqual(expected_author, actual_author)
class ParseAuthorTest(TestCase):
- @istest
- def author_basic(self):
+ def test_author_basic(self):
data = {
'author': "i-am-groot",
'author_email': 'iam@groot.org',
}
actual_author = author(data)
expected_author = {
'fullname': b'i-am-groot ',
'name': b'i-am-groot',
'email': b'iam@groot.org',
}
self.assertEquals(actual_author, expected_author)
- @istest
- def author_malformed(self):
+ def test_author_malformed(self):
data = {
'author': "['pierre', 'paul', 'jacques']",
'author_email': None,
}
actual_author = author(data)
expected_author = {
'fullname': b"['pierre', 'paul', 'jacques']",
'name': b"['pierre', 'paul', 'jacques']",
'email': None,
}
self.assertEquals(actual_author, expected_author)
- @istest
- def author_malformed_2(self):
+ def test_author_malformed_2(self):
data = {
'author': '[marie, jeanne]',
'author_email': '[marie@some, jeanne@thing]',
}
actual_author = author(data)
expected_author = {
'fullname': b'[marie, jeanne] <[marie@some, jeanne@thing]>',
'name': b'[marie, jeanne]',
'email': b'[marie@some, jeanne@thing]',
}
self.assertEquals(actual_author, expected_author)
- @istest
- def author_malformed_3(self):
+ def test_author_malformed_3(self):
data = {
'author': '[marie, jeanne, pierre]',
'author_email': '[marie@somewhere.org, jeanne@somewhere.org]',
}
actual_author = author(data)
expected_author = {
'fullname': b'[marie, jeanne, pierre] <[marie@somewhere.org, jeanne@somewhere.org]>', # noqa
'name': b'[marie, jeanne, pierre]',
'email': b'[marie@somewhere.org, jeanne@somewhere.org]',
}
self.assertEquals(actual_author, expected_author)
diff --git a/swh/loader/pypi/tests/test_loader.py b/swh/loader/pypi/tests/test_loader.py
index 2b521f3..4849489 100644
--- a/swh/loader/pypi/tests/test_loader.py
+++ b/swh/loader/pypi/tests/test_loader.py
@@ -1,556 +1,475 @@
# Copyright (C) 2016-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
import shutil
import tempfile
from nose.plugins.attrib import attr
-from nose.tools import istest
-from unittest import TestCase
-
-from swh.model import hashutil
+from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage
from swh.loader.pypi.client import PyPIProject
from swh.loader.pypi.loader import PyPILoader
-from .common import PyPIClientWithCache, RESOURCES_PATH, LoaderNoStorage
+from swh.model import hashutil
+
+from .common import RESOURCES_PATH, PyPIClientWithCache
class TestPyPILoader(LoaderNoStorage, PyPILoader):
"""Real PyPILoader for test purposes (storage and pypi interactions
inhibited)
"""
def __init__(self, project_name, json_filename=None):
if not json_filename: # defaulting to using same name as project
json_filename = '%s.json' % project_name
project_metadata_file = '%s/%s' % (RESOURCES_PATH, json_filename)
project_metadata_url = 'https://pypi.org/pypi/%s/json' % project_name
with open(project_metadata_file) as f:
data = json.load(f)
- temp_dir = tempfile.mkdtemp(
+ self.temp_dir = tempfile.mkdtemp(
dir='/tmp/', prefix='swh.loader.pypi.tests-')
# Will use the pypi with cache
client = PyPIClientWithCache(
- temp_directory=temp_dir, cache_dir=RESOURCES_PATH)
+ temp_directory=self.temp_dir, cache_dir=RESOURCES_PATH)
super().__init__(client=client)
self.project = PyPIProject(
client=client,
project=project_name,
project_metadata_url=project_metadata_url,
data=data)
def prepare(self, project_name, origin_url,
origin_metadata_url=None):
self.project_name = project_name
self.origin_url = origin_url
self.origin_metadata_url = origin_metadata_url
self.visit = 1 # first visit
self._prepare_state()
@attr('fs')
-class BaseLoaderITest(TestCase):
+class PyPIBaseLoaderTest(BaseLoaderTest):
"""Loader Test Mixin to prepare the pypi to 'load' in a test context.
In this setup, the loader uses the cache to load data so no
network interaction (no storage, no pypi).
"""
def setUp(self, project_name='0805nexter',
dummy_pypi_instance='https://dummy.org'):
- self.tmp_root_path = tempfile.mkdtemp()
+ self.tmp_root_path = tempfile.mkdtemp(
+ dir='/tmp', prefix='swh.loader.pypi.tests-')
self._project = project_name
self._origin_url = '%s/pypi/%s/' % (dummy_pypi_instance, project_name)
self._project_metadata_url = '%s/pypi/%s/json' % (
dummy_pypi_instance, project_name)
- def tearDown(self):
- shutil.rmtree(self.tmp_root_path)
-
- def assertContentsOk(self, expected_contents):
- contents = self.loader.all_contents
- self.assertEquals(len(contents), len(expected_contents))
-
- for content in contents:
- content_id = hashutil.hash_to_hex(content['sha1'])
- self.assertIn(content_id, expected_contents)
-
- def assertDirectoriesOk(self, expected_directories):
- directories = self.loader.all_directories
- self.assertEquals(len(directories), len(expected_directories))
-
- for _dir in directories:
- _dir_id = hashutil.hash_to_hex(_dir['id'])
- self.assertIn(_dir_id, expected_directories)
-
- def assertSnapshotOk(self, expected_snapshot, expected_branches):
- snapshots = self.loader.all_snapshots
- self.assertEqual(len(snapshots), 1)
-
- snap = snapshots[0]
- snap_id = hashutil.hash_to_hex(snap['id'])
- self.assertEqual(snap_id, expected_snapshot)
-
- def decode_target(target):
- if not target:
- return target
- target_type = target['target_type']
-
- if target_type == 'alias':
- decoded_target = target['target'].decode('utf-8')
- else:
- decoded_target = hashutil.hash_to_hex(target['target'])
-
- return {
- 'target': decoded_target,
- 'target_type': target_type
- }
-
- branches = {
- branch.decode('utf-8'): decode_target(target)
- for branch, target in snap['branches'].items()
- }
- self.assertEqual(expected_branches, branches)
-
- def assertRevisionsOk(self, expected_revisions): # noqa: N802
- """Check the loader's revisions match the expected revisions.
-
- Expects self.loader to be instantiated and ready to be
- inspected (meaning the loading took place).
-
- Args:
- expected_revisions (dict): Dict with key revision id,
- value the targeted directory id.
-
- """
- # The last revision being the one used later to start back from
- for rev in self.loader.all_revisions:
- rev_id = hashutil.hash_to_hex(rev['id'])
- directory_id = hashutil.hash_to_hex(rev['directory'])
-
- self.assertEquals(expected_revisions[rev_id], directory_id)
-
-
-# Define loaders with no storage
-# They'll just accumulate the data in place
-# Only for testing purposes.
-
class PyPILoaderNoSnapshot(TestPyPILoader):
"""Same as TestPyPILoader with no prior snapshot seen
"""
def _last_snapshot(self):
return None
-class LoaderITest(BaseLoaderITest):
+class LoaderITest(PyPIBaseLoaderTest):
def setUp(self, project_name='0805nexter',
dummy_pypi_instance='https://dummy.org'):
super().setUp(project_name, dummy_pypi_instance)
self.loader = PyPILoaderNoSnapshot(project_name=project_name)
- @istest
- def load(self):
+ def test_load(self):
"""Load a pypi origin
"""
# when
self.loader.load(
self._project, self._origin_url, self._project_metadata_url)
# then
- self.assertEquals(len(self.loader.all_contents), 6,
- '3 contents per release artifact files (2)')
- self.assertEquals(len(self.loader.all_directories), 4)
- self.assertEquals(len(self.loader.all_revisions), 2,
- '2 releases so 2 revisions should be created')
- self.assertEquals(len(self.loader.all_releases), 0,
- 'No release is created in the pypi loader')
- self.assertEquals(len(self.loader.all_snapshots), 1,
- 'Only 1 snapshot targetting all revisions')
+ self.assertCountContents(
+ 6, '3 contents per release artifact files (2)')
+ self.assertCountDirectories(4)
+ self.assertCountRevisions(
+ 2, '2 releases so 2 revisions should be created')
+ self.assertCountReleases(0, 'No release is created in the pypi loader')
+ self.assertCountSnapshots(1, 'Only 1 snapshot targeting all revisions')
expected_contents = [
'a61e24cdfdab3bb7817f6be85d37a3e666b34566',
'938c33483285fd8ad57f15497f538320df82aeb8',
'a27576d60e08c94a05006d2e6d540c0fdb5f38c8',
'405859113963cb7a797642b45f171d6360425d16',
'e5686aa568fdb1d19d7f1329267082fe40482d31',
'83ecf6ec1114fd260ca7a833a2d165e71258c338',
]
self.assertContentsOk(expected_contents)
expected_directories = [
'05219ba38bc542d4345d5638af1ed56c7d43ca7d',
'cf019eb456cf6f78d8c4674596f1c9a97ece8f44',
'b178b66bd22383d5f16f4f5c923d39ca798861b4',
'c3a58f8b57433a4b56caaa5033ae2e0931405338',
]
self.assertDirectoriesOk(expected_directories)
# {revision hash: directory hash}
expected_revisions = {
'4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa
'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa
}
self.assertRevisionsOk(expected_revisions)
expected_branches = {
'releases/1.1.0': {
'target': '4c99891f93b81450385777235a37b5e966dd1571',
'target_type': 'revision',
},
'releases/1.2.0': {
'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
'target_type': 'revision',
},
'HEAD': {
'target': 'releases/1.2.0',
'target_type': 'alias',
},
}
self.assertSnapshotOk('ba6e158ada75d0b3cfb209ffdf6daa4ed34a227a',
expected_branches)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')
class PyPILoaderWithSnapshot(TestPyPILoader):
"""This loader provides a snapshot and lists corresponding seen
release artifacts.
"""
def _last_snapshot(self):
"""Return last visited snapshot"""
return {
'id': b'\xban\x15\x8a\xdau\xd0\xb3\xcf\xb2\t\xff\xdfm\xaaN\xd3J"z', # noqa
'branches': {
b'releases/1.1.0': {
'target': b'L\x99\x89\x1f\x93\xb8\x14P'
b'8Ww#Z7\xb5\xe9f\xdd\x15q',
'target_type': 'revision'
},
b'releases/1.2.0': {
'target': b'\xe4E\xdaM\xa2+1\xbf'
b'\xeb\xb6\xff\xc48=\xbf\x83'
b'\x9a\x07M!',
'target_type': 'revision'
},
b'HEAD': {
'target': b'releases/1.2.0',
'target_type': 'alias'
},
},
}
def _known_artifacts(self, last_snapshot):
"""List corresponding seen release artifacts"""
return {
(
'0805nexter-1.1.0.zip',
'52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035' # noqa
): b'L\x99\x89\x1f\x93\xb8\x14P8Ww#Z7\xb5\xe9f\xdd\x15q',
(
'0805nexter-1.2.0.zip',
'49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709' # noqa
): b'\xe4E\xdaM\xa2+1\xbf\xeb\xb6\xff\xc48=\xbf\x83\x9a\x07M!',
}
-class LoaderNoNewChangesSinceLastVisitITest(BaseLoaderITest):
+class LoaderNoNewChangesSinceLastVisitITest(PyPIBaseLoaderTest):
"""This scenario makes use of the incremental nature of the loader.
If nothing changes in between visits, the snapshot for the visit
must stay the same as the first visit.
"""
def setUp(self, project_name='0805nexter',
dummy_pypi_instance='https://dummy.org'):
super().setUp(project_name, dummy_pypi_instance)
self.loader = PyPILoaderWithSnapshot(project_name=project_name)
- @istest
- def load(self):
+ def test_load(self):
"""Load a PyPI origin without new changes results in 1 same snapshot
"""
# when
self.loader.load(
self._project, self._origin_url, self._project_metadata_url)
# then
- self.assertEquals(len(self.loader.all_contents), 0)
- self.assertEquals(len(self.loader.all_directories), 0)
- self.assertEquals(len(self.loader.all_revisions), 0)
- self.assertEquals(len(self.loader.all_releases), 0)
- self.assertEquals(len(self.loader.all_snapshots), 1)
+ self.assertCountContents(0)
+ self.assertCountDirectories(0)
+ self.assertCountRevisions(0)
+ self.assertCountReleases(0)
+ self.assertCountSnapshots(1)
self.assertContentsOk([])
self.assertDirectoriesOk([])
self.assertRevisionsOk(expected_revisions={})
expected_snapshot_id = 'ba6e158ada75d0b3cfb209ffdf6daa4ed34a227a'
expected_branches = {
'releases/1.1.0': {
'target': '4c99891f93b81450385777235a37b5e966dd1571',
'target_type': 'revision',
},
'releases/1.2.0': {
'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
'target_type': 'revision',
},
'HEAD': {
'target': 'releases/1.2.0',
'target_type': 'alias',
},
}
self.assertSnapshotOk(expected_snapshot_id, expected_branches)
_id = hashutil.hash_to_hex(self.loader._last_snapshot()['id'])
self.assertEquals(expected_snapshot_id, _id)
self.assertEqual(self.loader.load_status(), {'status': 'uneventful'})
self.assertEqual(self.loader.visit_status(), 'full')
-class LoaderNewChangesSinceLastVisitITest(BaseLoaderITest):
+class LoaderNewChangesSinceLastVisitITest(PyPIBaseLoaderTest):
"""In this scenario, a visit has already taken place.
An existing snapshot exists.
This time, the PyPI project has changed, a new release (with 1 new
release artifact) has been uploaded. The old releases did not
change.
The visit results in a new snapshot.
The new snapshot shares the same history as prior visit's
- snapshot. It holds a new branch targetting the new revision.
+ snapshot. It holds a new branch targeting the new revision.
"""
def setUp(self, project_name='0805nexter',
dummy_pypi_instance='https://dummy.org'):
super().setUp(project_name, dummy_pypi_instance)
self.loader = PyPILoaderWithSnapshot(
project_name=project_name,
json_filename='0805nexter+new-made-up-release.json')
- @istest
- def load(self):
+ def test_load(self):
"""Load a PyPI origin with changes results in 1 new snapshot
"""
# when
self.loader.load(
self._project, self._origin_url, self._project_metadata_url)
# then
- self.assertEquals(
- len(self.loader.all_contents), 4,
+ self.assertCountContents(4,
"3 + 1 new content (only change between 1.2.0 and 1.3.0 archives)")
- self.assertEquals(len(self.loader.all_directories), 2)
- self.assertEquals(
- len(self.loader.all_revisions), 1,
- "This results in 1 new revision targetting that new directory id")
- self.assertEquals(len(self.loader.all_releases), 0)
- self.assertEquals(len(self.loader.all_snapshots), 1)
+ self.assertCountDirectories(2)
+ self.assertCountRevisions(
+ 1, "1 new revision targeting that new directory id")
+ self.assertCountReleases(0)
+ self.assertCountSnapshots(1)
expected_contents = [
'92689fa2b7fb4d4fc6fb195bf73a50c87c030639', # new one
'405859113963cb7a797642b45f171d6360425d16',
'83ecf6ec1114fd260ca7a833a2d165e71258c338',
'e5686aa568fdb1d19d7f1329267082fe40482d31',
]
self.assertContentsOk(expected_contents)
expected_directories = [
'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a',
'52604d46843b898f5a43208045d09fcf8731631b',
]
self.assertDirectoriesOk(expected_directories)
expected_revisions = {
'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d': 'e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a', # noqa
}
self.assertRevisionsOk(expected_revisions)
old_revisions = {
'4c99891f93b81450385777235a37b5e966dd1571': '05219ba38bc542d4345d5638af1ed56c7d43ca7d', # noqa
'e445da4da22b31bfebb6ffc4383dbf839a074d21': 'b178b66bd22383d5f16f4f5c923d39ca798861b4', # noqa
}
for rev, dir_id in old_revisions.items():
expected_revisions[rev] = dir_id
expected_snapshot_id = '07322209e51618410b5e43ca4af7e04fe5113c9d'
expected_branches = {
'releases/1.1.0': {
'target': '4c99891f93b81450385777235a37b5e966dd1571',
'target_type': 'revision',
},
'releases/1.2.0': {
'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
'target_type': 'revision',
},
'releases/1.3.0': {
'target': 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d',
'target_type': 'revision',
},
'HEAD': {
'target': 'releases/1.3.0',
'target_type': 'alias',
},
}
self.assertSnapshotOk(expected_snapshot_id, expected_branches)
_id = hashutil.hash_to_hex(self.loader._last_snapshot()['id'])
self.assertNotEqual(expected_snapshot_id, _id)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')
class PyPILoaderWithSnapshot2(TestPyPILoader):
"""This loader provides a snapshot and lists corresponding seen
release artifacts.
"""
def _last_snapshot(self):
"""Return last visited snapshot"""
return {
'id': b'\x072"\t\xe5\x16\x18A\x0b^C\xcaJ\xf7\xe0O\xe5\x11<\x9d', # noqa
'branches': {
b'releases/1.1.0': {
'target': b'L\x99\x89\x1f\x93\xb8\x14P8Ww#Z7\xb5\xe9f\xdd\x15q', # noqa
'target_type': 'revision'
},
b'releases/1.2.0': {
'target': b'\xe4E\xdaM\xa2+1\xbf\xeb\xb6\xff\xc48=\xbf\x83\x9a\x07M!', # noqa
'target_type': 'revision'
},
b'releases/1.3.0': {
'target': b'\xfbF\xe4\x96\x05\xb0\xbb\xe6\x9f\x8cS\xd3\x15\xe8\x93p\xe7\xc6\xcb]', # noqa
'target_type': 'revision'
},
b'HEAD': {
'target': b'releases/1.3.0', # noqa
'target_type': 'alias'
},
}
}
def _known_artifacts(self, last_snapshot):
"""Map previously seen release artifacts to their revision"""
return {
(
'0805nexter-1.1.0.zip',
'52cd128ad3afe539478abc7440d4b043384295fbe6b0958a237cb6d926465035' # noqa
): b'L\x99\x89\x1f\x93\xb8\x14P8Ww#Z7\xb5\xe9f\xdd\x15q',
(
'0805nexter-1.2.0.zip',
'49785c6ae39ea511b3c253d7621c0b1b6228be2f965aca8a491e6b84126d0709' # noqa
): b'\xe4E\xdaM\xa2+1\xbf\xeb\xb6\xff\xc48=\xbf\x83\x9a\x07M!',
(
'0805nexter-1.3.0.zip',
'7097c49fb8ec24a7aaab54c3dbfbb5a6ca1431419d9ee0f6c363d9ad01d2b8b1' # noqa
): b'\xfbF\xe4\x96\x05\xb0\xbb\xe6\x9f\x8cS\xd3\x15\xe8\x93p\xe7\xc6\xcb]', # noqa
}
-class LoaderChangesOldReleaseArtifactRemovedSinceLastVisit(BaseLoaderITest):
+class LoaderChangesOldReleaseArtifactRemovedSinceLastVisit(PyPIBaseLoaderTest):
"""In this scenario, a visit has already taken place. An existing
snapshot exists.
The PyPI project has changed:
- a new release has been uploaded
- an older one has been removed
The visit should result in a new snapshot. Such snapshot shares some of
the same branches as prior visit (but not all):
- new release artifact branch exists
- old release artifact branch has been removed
- the other unchanged release artifact branches are left unchanged
"""
def setUp(self, project_name='0805nexter',
dummy_pypi_instance='https://dummy.org'):
super().setUp(project_name, dummy_pypi_instance)
self.loader = PyPILoaderWithSnapshot2(
project_name=project_name,
json_filename='0805nexter-unpublished-release.json')
- @istest
- def load(self):
+ def test_load(self):
"""Load PyPI origin with removed artifact + changes ~> 1 new snapshot
"""
# when
self.loader.load(
self._project, self._origin_url, self._project_metadata_url)
# then
- self.assertEquals(
- len(self.loader.all_contents), 4,
+ self.assertCountContents(4,
"3 + 1 new content (only change between 1.3.0 and 1.4.0 archives)")
- self.assertEquals(len(self.loader.all_directories), 2)
- self.assertEquals(
- len(self.loader.all_revisions), 1,
- "This results in 1 new revision targetting that new directory id")
- self.assertEquals(len(self.loader.all_releases), 0)
- self.assertEquals(len(self.loader.all_snapshots), 1)
+ self.assertCountDirectories(2)
+ self.assertCountRevisions(1,
+ "This results in 1 new revision targeting that new directory id")
+ self.assertCountReleases(0)
+ self.assertCountSnapshots(1)
expected_contents = [
'e2d68a197e3a3ad0fc6de28749077892c2148043', # new one
'405859113963cb7a797642b45f171d6360425d16',
'83ecf6ec1114fd260ca7a833a2d165e71258c338',
'e5686aa568fdb1d19d7f1329267082fe40482d31',
]
self.assertContentsOk(expected_contents)
expected_directories = [
'a2b7621f3e52eb3632657f6e3436bd08202db56f', # new one
'770e21215ecac53cea331d8ea4dc0ffc9d979367',
]
self.assertDirectoriesOk(expected_directories)
expected_revisions = {
# 1.4.0
'5e91875f096ac48c98d74acf307439a3490f2827': '770e21215ecac53cea331d8ea4dc0ffc9d979367', # noqa
}
self.assertRevisionsOk(expected_revisions)
expected_snapshot_id = 'bb0b0c29040678eadb6dae9e43e496cc860123e4'
expected_branches = {
'releases/1.2.0': {
'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
'target_type': 'revision',
},
'releases/1.3.0': {
'target': 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d',
'target_type': 'revision',
},
'releases/1.4.0': {
'target': '5e91875f096ac48c98d74acf307439a3490f2827',
'target_type': 'revision',
},
'HEAD': {
'target': 'releases/1.4.0',
'target_type': 'alias',
},
}
self.assertSnapshotOk(expected_snapshot_id, expected_branches)
_id = hashutil.hash_to_hex(self.loader._last_snapshot()['id'])
self.assertNotEqual(expected_snapshot_id, _id)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')
diff --git a/version.txt b/version.txt
index 5782386..9d293ed 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-v0.0.2-0-gf140bd0
\ No newline at end of file
+v0.0.3-0-gb237da9
\ No newline at end of file