diff --git a/swh/loader/package/npm.py b/swh/loader/package/npm.py --- a/swh/loader/package/npm.py +++ b/swh/loader/package/npm.py @@ -85,12 +85,17 @@ i_metadata = extract_intrinsic_metadata(uncompressed_path) # from intrinsic metadata author = extract_npm_package_author(i_metadata) - # extrinsic metadata - version = i_metadata['version'] - date = self.info['time'][version] + message = i_metadata['version'].encode('ascii') + + # from extrinsic metadata + + # No date available in intrinsic metadata: retrieve it from the API + # metadata, using the version number that the API claims this package + # has. + extrinsic_version = a_metadata['version'] + date = self.info['time'][extrinsic_version] date = iso8601.parse_date(date) date = normalize_timestamp(int(date.timestamp())) - message = version.encode('ascii') return { 'type': 'tar', diff --git a/swh/loader/package/tests/data/https_registry.npmjs.org/@aller_shared_-_shared-0.1.0.tgz b/swh/loader/package/tests/data/https_registry.npmjs.org/@aller_shared_-_shared-0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 GIT binary patch literal 0 Hc$@=10.0.0" + }, + "dependencies": { + "@aller/blink-labrador": "8.2.2", + "@aller/shiny": "^0.47.0", + "@apollo/react-hooks": "3.1.1", + "@babel/core": "7.6.2", + "@firebase/app-types": "0.x", + "@firebase/util": "0.x", + "@godaddy/terminus": "4.2.0", + "@material-ui/core": "4.4.3", + "@material-ui/lab": "3.0.0-alpha.30", + "@zeit/next-bundle-analyzer": "0.1.2", + "acorn": "7.1.0", + "adblock-detect": "1.0.5", + "apollo-cache-inmemory": "^1.6.3", + "apollo-client": "^2.6.4", + "apollo-link": "^1.2.13", + "apollo-link-error": "^1.1.12", + "apollo-link-http": "^1.5.16", + "arcads": "1.4.5", + "aurora-deep-slice-merge": "github:soldotno/aurora-deep-slice-merge", + "axios": "0.19.0", + "babel-plugin-styled-components": "1.10.6", + "circular-json": "0.5.9", + "classnames": "2.2.6", + "compression": "1.7.4", + "copy-webpack-plugin": "5.0.4", + "date-fns": "2.3.0", + "debug": "3.2.6", + "dotenv": "8.1.0", + "express": "4.17.1", + "express-prom-bundle": "5.1.5", + "firebase": "7.0.0", + "graphql": "14.5.8", + "graphql-tag": "^2.10.1", + "hoist-non-react-statics": "3.3.0", + "intersection-observer": "0.7.0", + "is-client": "0.0.2", + "isomorphic-unfetch": "3.0.0", + "jaeger-client": "3.17.0", + "js-cookie": "2.2.1", + "lazysizes": "5.1.1", + "lodash": "4.17.15", + "lodash.debounce": "4.0.8", + "lodash.isinteger": "4.0.4", + "lodash.throttle": "4.1.1", + "next": "9.0.5", + "next-transpile-modules": "2.3.1", + "nodesi": "1.11.0", + "on-idle": "3.1.4", + "opentracing": "0.14.4", + "ot-got": "0.3.1", + "polished": "3.4.1", + "prebid.js": "^2.27.0", + "prom-client": "11.5.3", + "prop-types": "15.7.2", + "query-string": "5.1.1", + "react": "16.9.0", + "react-checkbox-group": "4.0.1", + "react-content-loader": "^4.2.2", + "react-dom": "16.9.0", + "react-gpt": "2.0.1", + "react-instagram-embed": "^1.5.0", + "react-intersection-observer": "8.24.2", + "react-prebid": "2.1.1", + "react-redux": "7.1.1", + "redux": "4.0.4", + "redux-thunk": "2.3.0", + "reselect": "4.0.0", + "serve-static": "1.14.1", + "sol-menu-content": "1.0.10", + "storybook-chromatic": "2.2.2", + "striptags": "3.1.1", + "styled-components": "5.0.0-beta.8", + "ucfirst": "1.0.0", + "ulog": "beta", + "victory-pie": "33.1.0", + "victory-tooltip": "33.1.0" + }, + "_id": "@aller/shared@0.1.0", + "dist": { + "shasum": "8231e9ad6d175843e1022cd7d0eb0ad6b6087603", + "integrity": "sha512-I2QHU6gHft3QN50wTZhNjzAesXbTEmdX/G3F5ID0+s7VXC7Z3EsUP9Abx2/G78JGINfe4mCqHRXPgHn9z9JKRA==", + "tarball": "https://registry.npmjs.org/@aller/shared/-/shared-0.1.0.tgz", + "fileCount": 686, + "unpackedSize": 6182869 + }, + "maintainers": [], + "directories": null, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/shared_0.1.0_1570023680880_0.29907609962091875" + }, + "_hasShrinkwrap": false + }, + "0.1.1-alpha.14": { + "name": "@aller/shared", + "version": "0.1.1-alpha.14", + "description": "Library of shared code", + "license": "UNLICENSED", + "scripts": { + "build": "yarn build-storybook", + "build-storybook": "build-storybook -c .storybook -o storybook-static", + "lint": "yarn lint:eslint", + "lint:fix": "yarn lint:eslint:fix", + "lint:eslint": "eslint --max-warnings=0 .", + "lint:eslint:fix": "yarn lint:eslint --fix", + "lint:eslint:junit": "yarn lint:eslint --format junit -o reports/junit/js-lint-results.xml", + "storybook": "start-storybook -p 9001 -c .storybook", + "test": "yarn lint:eslint && yarn test:jest", + "test:ci": "yarn lint:eslint:junit && yarn test:jest:junit", + "test:jest": "NODE_ENV=test jest", + "test:jest:junit": "yarn test:jest --ci --runInBand --reporters=default --reporters=jest-junit", + "gql:generate": "graphql-codegen", + "gql:watch": "graphql-codegen --watch", + "gql:init": "graphql-codegen init" + }, + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@graphql-codegen/cli": "1.7.0", + "@graphql-codegen/fragment-matcher": "^1.7.0", + "@graphql-codegen/typescript": "1.7.0", + "@graphql-codegen/typescript-operations": "1.7.0", + "@graphql-codegen/typescript-react-apollo": "1.7.0", + "@storybook/addon-a11y": "5.2.1", + "@storybook/addon-actions": "5.2.1", + "@storybook/addon-contexts": "^5.1.11", + "@storybook/addon-links": "5.2.1", + "@storybook/addon-viewport": "5.2.1", + "@storybook/addons": "5.2.1", + "@storybook/react": "5.2.1", + "@storybook/theming": "5.2.1", + "fs-extra": "^8.1.0", + "glob": "^7.1.3", + "ts-jest": "^24.1.0" + }, + "main": "./src/index", + "module": "./src/index", + "sideEffects": false, + "engines": { + "node": ">=10.0.0" + }, + "dependencies": { + "@aller/blink-labrador": "8.2.2", + "@aller/shiny": "^0.47.0", + "@apollo/react-hooks": "3.1.1", + "@babel/core": "7.6.2", + "@firebase/app-types": "0.x", + "@firebase/util": "0.x", + "@godaddy/terminus": "4.2.0", + "@material-ui/core": "4.4.3", + "@material-ui/lab": "3.0.0-alpha.30", + "@zeit/next-bundle-analyzer": "0.1.2", + "acorn": "7.1.0", + "adblock-detect": "1.0.5", + "apollo-cache-inmemory": "^1.6.3", + "apollo-client": "^2.6.4", + "apollo-link": "^1.2.13", + "apollo-link-error": "^1.1.12", + "apollo-link-http": "^1.5.16", + "arcads": "1.4.5", + "aurora-deep-slice-merge": "github:soldotno/aurora-deep-slice-merge", + "axios": "0.19.0", + "babel-plugin-styled-components": "1.10.6", + "circular-json": "0.5.9", + "classnames": "2.2.6", + "compression": "1.7.4", + "copy-webpack-plugin": "5.0.4", + "date-fns": "2.3.0", + "debug": "3.2.6", + "dotenv": "8.1.0", + "express": "4.17.1", + "express-prom-bundle": "5.1.5", + "firebase": "7.0.0", + "graphql": "14.5.8", + "graphql-tag": "^2.10.1", + "hoist-non-react-statics": "3.3.0", + "intersection-observer": "0.7.0", + "is-client": "0.0.2", + "isomorphic-unfetch": "3.0.0", + "jaeger-client": "3.17.0", + "js-cookie": "2.2.1", + "lazysizes": "5.1.1", + "lodash": "4.17.15", + "lodash.debounce": "4.0.8", + "lodash.isinteger": "4.0.4", + "lodash.throttle": "4.1.1", + "next": "9.0.5", + "next-transpile-modules": "2.3.1", + "nodesi": "1.11.0", + "on-idle": "3.1.4", + "opentracing": "0.14.4", + "ot-got": "0.3.1", + "polished": "3.4.1", + "prebid.js": "^2.27.0", + "prom-client": "11.5.3", + "prop-types": "15.7.2", + "query-string": "5.1.1", + "react": "16.9.0", + "react-checkbox-group": "4.0.1", + "react-content-loader": "^4.2.2", + "react-dom": "16.9.0", + "react-gpt": "2.0.1", + "react-instagram-embed": "^1.5.0", + "react-intersection-observer": "8.24.2", + "react-prebid": "2.1.1", + "react-redux": "7.1.1", + "redux": "4.0.4", + "redux-thunk": "2.3.0", + "reselect": "4.0.0", + "serve-static": "1.14.1", + "sol-menu-content": "1.0.10", + "storybook-chromatic": "2.2.2", + "striptags": "3.1.1", + "styled-components": "5.0.0-beta.8", + "ucfirst": "1.0.0", + "ulog": "beta", + "victory-pie": "33.1.0", + "victory-tooltip": "33.1.0" + }, + "gitHead": "ac08282d37c72abf3f7c7b1238edb6a653195646", + "readme": "ERROR: No README data found!", + "_id": "@aller/shared@0.1.1-alpha.14", + "_nodeVersion": "10.16.3", + "_npmVersion": "lerna/3.16.4/node@v10.16.3+x64 (linux)", + "dist": { + "integrity": "sha512-WDgwdL76W6aV44dw+asf+QA1uWHVCt0074WJSVA9VxVWIjL5S4Dg8+b5GnU9047KalLW8fy53PlOQqClJ3qJ1A==", + "shasum": "b36f8ea2d17305ce1110fb096d23d8abf948ff6d", + "tarball": "https://registry.npmjs.org/@aller/shared/-/shared-0.1.1-alpha.14.tgz", + "fileCount": 504, + "unpackedSize": 850607 + }, + "maintainers": [], + "directories": null, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/shared_0.1.1-alpha.14_1570024938845_0.37086021016657233" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2019-10-01T18:57:21.566Z", + "modified": "2019-11-07T14:11:06.245Z", + "0.1.0": "2019-10-02T13:41:21.150Z", + "0.1.1-alpha.14": "2019-10-02T14:02:19.056Z" + }, + "maintainers": [], + "description": "Library of shared code", + "license": "UNLICENSED", + "readme": "", + "readmeFilename": "" +} diff --git a/swh/loader/package/tests/test_npm.py b/swh/loader/package/tests/test_npm.py --- a/swh/loader/package/tests/test_npm.py +++ b/swh/loader/package/tests/test_npm.py @@ -6,6 +6,7 @@ import json import os +import pytest from swh.model.hashutil import hash_to_bytes @@ -534,3 +535,50 @@ if m.url.startswith('https://registry.npmjs.org') ] assert len(urls) == len(set(urls)) # we visited each artifact once across + + +@pytest.mark.usefixtures('requests_mock_datadir') +def test_npm_loader_version_divergence(swh_config): + package = '@aller_shared' + url = package_url(package) + loader = NpmLoader(package, url, package_metadata_url(package)) + + actual_load_status = loader.load() + assert actual_load_status['status'] == 'eventful' + assert actual_load_status['status'] is not None + origin_visit = list(loader.storage.origin_visit_get(url))[-1] + assert origin_visit['status'] == 'full' + assert origin_visit['type'] == 'npm' + + stats = get_stats(loader.storage) + + assert { # 1 new releases artifacts + 'content': 534, + 'directory': 153, + 'origin': 1, + 'origin_visit': 1, + 'person': 1, + 'release': 0, + 'revision': 2, + 'skipped_content': 0, + 'snapshot': 1, + } == stats + + expected_snapshot = { + 'id': 'b11ebac8c9d0c9e5063a2df693a18e3aba4b2f92', + 'branches': { + 'HEAD': { + 'target_type': 'alias', + 'target': 'releases/0.1.0' + }, + 'releases/0.1.0': { + 'target_type': 'revision', + 'target': '845673bfe8cbd31b1eaf757745a964137e6f9116', + }, + 'releases/0.1.1-alpha.14': { + 'target_type': 'revision', + 'target': '05181c12cd8c22035dd31155656826b85745da37', + }, + }, + } + check_snapshot(expected_snapshot, loader.storage)