diff --git a/Makefile.local b/Makefile.local --- a/Makefile.local +++ b/Makefile.local @@ -2,6 +2,7 @@ TESTFLAGS = --hypothesis-profile=swh-web-fast TESTFULL_FLAGS = --hypothesis-profile=swh-web YARN ?= yarn +HTTP_PORT = 5004 yarn-install: package.json $(YARN) install @@ -33,32 +34,32 @@ echo "flush_all" | nc -q 2 localhost 11211 2>/dev/null run-django-webpack-devserver: run-migrations yarn-install - bash -c "trap 'trap - SIGINT SIGTERM ERR; kill %1' SIGINT SIGTERM ERR; $(YARN) start-dev & sleep 10 && cd swh/web && python3 manage.py runserver --nostatic" + bash -c "trap 'trap - SIGINT SIGTERM ERR; kill %1' SIGINT SIGTERM ERR; $(YARN) start-dev & sleep 10 && cd swh/web && python3 manage.py runserver --nostatic $(HTTP_PORT)" run-django-webpack-dev: build-webpack-dev run-migrations - python3 swh/web/manage.py runserver --nostatic + python3 swh/web/manage.py runserver --nostatic $(HTTP_PORT) run-django-webpack-prod: build-webpack-prod run-migrations-prod clear-memcached - python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production + python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production $(HTTP_PORT) run-django-server-dev: run-migrations - python3 swh/web/manage.py runserver --nostatic + python3 swh/web/manage.py runserver --nostatic $(HTTP_PORT) run-django-server-prod: run-migrations-prod clear-memcached - python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production + python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production $(HTTP_PORT) run-gunicorn-server: run-migrations clear-memcached gunicorn3 -b 127.0.0.1:5004 swh.web.wsgi run-django-webpack-memory-storages: build-webpack-dev run-migrations - python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests + python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests $(HTTP_PORT) test-full: $(TEST) $(TESTFULL_FLAGS) $(TEST_DIRS) test-frontend: build-webpack-dev run-migrations - python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & sleep 10 && $(YARN) run cypress run && pkill -P $$! + python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests $(HTTP_PORT) & sleep 10 && $(YARN) run cypress run && pkill -P $$! test-frontend-ui: build-webpack-dev run-migrations - bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT; jobs -p | head -1 | xargs pkill -P' SIGINT SIGTERM ERR EXIT; python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & sleep 10 && $(YARN) run cypress open" + bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT; jobs -p | head -1 | xargs pkill -P' SIGINT SIGTERM ERR EXIT; python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.test $(HTTP_PORT) & sleep 10 && $(YARN) run cypress open" diff --git a/cypress.json b/cypress.json --- a/cypress.json +++ b/cypress.json @@ -1,5 +1,5 @@ { - "baseUrl": "http://localhost:5004", + "baseUrl": "http://localhost:8080", "video": false, "viewportWidth": 1920, "viewportHeight": 1080 diff --git a/cypress/integration/origin-search.spec.js b/cypress/integration/origin-search.spec.js --- a/cypress/integration/origin-search.spec.js +++ b/cypress/integration/origin-search.spec.js @@ -77,7 +77,7 @@ .should('be.visible'); cy.contains('tr', archivedRepo.url) .should('be.visible') - .children('#visit-status-origin-2') + .children('#visit-status-origin-0') .children('i') .should('have.class', 'fa-check') .and('have.attr', 'title', diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -478,6 +478,48 @@ return result +@api_route(r'/origin/(?P.*)/visit/latest/', + 'api-1-origin-visit-latest') +@api_doc('/origin/visit/') +@format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) +def api_origin_visit_latest(request, origin_url=None): + """ + .. http:get:: /api/1/origin/(origin_url)/visit/latest/ + + Get information about a specific visit of a software origin. + + :param str origin_url: a software origin URL + :query boolean require_snapshot: if true, only return a visit + with a snapshot + + {common_headers} + + {return_origin_visit} + + **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, + :http:method:`options` + + :statuscode 200: no error + :statuscode 404: requested origin or visit can not be found in the + archive + + **Example:** + + .. parsed-literal:: + + :swh_web_api:`origin/https://github.com/hylang/hy/visit/latest/` + """ + require_snapshot = request.query_params.get('require_snapshot', 'false') + return api_lookup( + service.lookup_origin_visit_latest, origin_url, + bool(strtobool(require_snapshot)), + notfound_msg=('No visit for origin {} found' + .format(origin_url)), + enrich_fn=partial(_enrich_origin_visit, + with_origin_link=True, + with_origin_visit_link=False)) + + @api_route(r'/origin/(?P.*)/visit/(?P[0-9]+)/', 'api-1-origin-visit') @api_route(r'/origin/(?P[0-9]+)/visit/(?P[0-9]+)/', diff --git a/swh/web/assets/src/bundles/browse/origin-search.js b/swh/web/assets/src/bundles/browse/origin-search.js --- a/swh/web/assets/src/bundles/browse/origin-search.js +++ b/swh/web/assets/src/bundles/browse/origin-search.js @@ -25,35 +25,35 @@ $('#origin-search-results tbody tr').remove(); } -function populateOriginSearchResultsTable(data, offset) { +function populateOriginSearchResultsTable(origins, offset) { let localOffset = offset % limit; - if (data.length > 0) { + if (origins.length > 0) { $('#swh-origin-search-results').show(); $('#swh-no-result').hide(); clearOriginSearchResultsTable(); let table = $('#origin-search-results tbody'); - for (let i = localOffset; i < localOffset + perPage && i < data.length; ++i) { - let elem = data[i]; - let browseUrl = Urls.browse_origin(elem.url); - let tableRow = ``; - tableRow += `${elem.type}`; - tableRow += `${encodeURI(elem.url)}`; - tableRow += ``; + for (let i = localOffset; i < localOffset + perPage && i < origins.length; ++i) { + let origin = origins[i]; + let browseUrl = Urls.browse_origin(origin.url); + let tableRow = ``; + tableRow += `${origin.type}`; + tableRow += `${encodeURI(origin.url)}`; + tableRow += ``; tableRow += ''; table.append(tableRow); // get async latest visit snapshot and update visit status icon - let latestSnapshotUrl = Urls.browse_origin_latest_snapshot(elem.id); + let latestSnapshotUrl = Urls.api_1_origin_visit_latest(origin.url); + latestSnapshotUrl += "?require_snapshot"; fetch(latestSnapshotUrl) .then(response => response.json()) .then(data => { - let originId = elem.id; - $(`#visit-status-origin-${originId}`).children().remove(); + $(`#visit-status-origin-${i}`).children().remove(); if (data) { - $(`#visit-status-origin-${originId}`).append(''); + $(`#visit-status-origin-${i}`).append(''); } else { - $(`#visit-status-origin-${originId}`).append(''); + $(`#visit-status-origin-${i}`).append(''); if ($('#swh-filter-empty-visits').prop('checked')) { - $(`#origin-${originId}`).remove(); + $(`#origin-${i}`).remove(); } } }); @@ -64,8 +64,8 @@ $('#swh-no-result').text('No origins matching the search criteria were found.'); $('#swh-no-result').show(); } - if (data.length - localOffset < perPage || - (data.length < limit && (localOffset + perPage) === data.length)) { + if (origins.length - localOffset < perPage || + (origins.length < limit && (localOffset + perPage) === origins.length)) { $('#origins-next-results-button').addClass('disabled'); } else { $('#origins-next-results-button').removeClass('disabled'); diff --git a/swh/web/browse/views/origin.py b/swh/web/browse/views/origin.py --- a/swh/web/browse/views/origin.py +++ b/swh/web/browse/views/origin.py @@ -277,24 +277,6 @@ return HttpResponse(results, content_type='application/json') -@browse_route(r'origin/(?P[0-9]+)/latest_snapshot/', - view_name='browse-origin-latest-snapshot') -def _origin_latest_snapshot(request, origin_id): - """ - Internal browse endpoint used to check if an origin has already - been visited by Software Heritage and has at least one full visit. - """ - result = \ - service.lookup_latest_origin_snapshot(int(origin_id), - allowed_statuses=['full', - 'partial']) - - result = json.dumps(result, sort_keys=True, indent=4, - separators=(',', ': ')) - - return HttpResponse(result, content_type='application/json') - - @browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/', r'origin/(?P.+)/', view_name='browse-origin') diff --git a/swh/web/common/service.py b/swh/web/common/service.py --- a/swh/web/common/service.py +++ b/swh/web/common/service.py @@ -852,6 +852,22 @@ yield converters.from_origin_visit(visit) +def lookup_origin_visit_latest(origin_url, require_snapshot): + """Return the origin's latest visit + + Args: + origin_url (str): origin to list visits for + require_snapshot (bool): filter out origins without a snapshot + + Returns: + The dict origin_visit concerned + + """ + visit = storage.origin_visit_get_latest( + origin_url, require_snapshot=require_snapshot) + return converters.from_origin_visit(visit) + + def lookup_origin_visit(origin_url, visit_id): """Return information about visit visit_id with origin origin. diff --git a/swh/web/tests/api/views/test_origin.py b/swh/web/tests/api/views/test_origin.py --- a/swh/web/tests/api/views/test_origin.py +++ b/swh/web/tests/api/views/test_origin.py @@ -201,6 +201,79 @@ self.assertEqual(rv.data, expected_visit) + @given(new_origin(), visit_dates(2), new_snapshots(1)) + def test_api_lookup_origin_visit_latest( + self, new_origin, visit_dates, new_snapshots): + + origin_id = self.storage.origin_add_one(new_origin) + new_origin['id'] = origin_id + visit_dates.sort() + visit_ids = [] + for i, visit_date in enumerate(visit_dates): + origin_visit = self.storage.origin_visit_add(origin_id, visit_date) + visit_ids.append(origin_visit['visit']) + + self.storage.snapshot_add([new_snapshots[0]]) + self.storage.origin_visit_update( + origin_id, visit_ids[0], + snapshot=new_snapshots[0]['id']) + + url = reverse('api-1-origin-visit-latest', + url_args={'origin_url': new_origin['url']}) + + rv = self.client.get(url) + self.assertEqual(rv.status_code, 200, rv.data) + self.assertEqual(rv['Content-Type'], 'application/json') + + expected_visit = self.origin_visit_get_by(origin_id, visit_ids[1]) + + origin_url = reverse('api-1-origin', + url_args={'origin_url': new_origin['url']}) + + expected_visit['origin'] = new_origin['url'] + expected_visit['origin_url'] = origin_url + expected_visit['snapshot_url'] = None + + self.assertEqual(rv.data, expected_visit) + + @given(new_origin(), visit_dates(2), new_snapshots(1)) + def test_api_lookup_origin_visit_latest_with_snapshot( + self, new_origin, visit_dates, new_snapshots): + origin_id = self.storage.origin_add_one(new_origin) + new_origin['id'] = origin_id + visit_dates.sort() + visit_ids = [] + for i, visit_date in enumerate(visit_dates): + origin_visit = self.storage.origin_visit_add(origin_id, visit_date) + visit_ids.append(origin_visit['visit']) + + self.storage.snapshot_add([new_snapshots[0]]) + self.storage.origin_visit_update( + origin_id, visit_ids[0], + snapshot=new_snapshots[0]['id']) + + url = reverse('api-1-origin-visit-latest', + url_args={'origin_url': new_origin['url']}) + url += '?require_snapshot=true' + + rv = self.client.get(url) + self.assertEqual(rv.status_code, 200, rv.data) + self.assertEqual(rv['Content-Type'], 'application/json') + + expected_visit = self.origin_visit_get_by(origin_id, visit_ids[0]) + + origin_url = reverse('api-1-origin', + url_args={'origin_url': new_origin['url']}) + snapshot_url = reverse( + 'api-1-snapshot', + url_args={'snapshot_id': expected_visit['snapshot']}) + + expected_visit['origin'] = new_origin['url'] + expected_visit['origin_url'] = origin_url + expected_visit['snapshot_url'] = snapshot_url + + self.assertEqual(rv.data, expected_visit) + @pytest.mark.origin_id @given(new_origin(), visit_dates(3), new_snapshots(3)) def test_api_lookup_origin_visit_by_id(self, new_origin, visit_dates,