diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js --- a/cypress/integration/origin-save.spec.js +++ b/cypress/integration/origin-save.spec.js @@ -17,7 +17,7 @@ }; function makeOriginSaveRequest(originType, originUrl) { - cy.get('#swh-input-origin-type') + cy.get('#swh-input-visit-type') .select(originType) .get('#swh-input-origin-url') .type(originUrl) @@ -47,7 +47,7 @@ // object_type : {'git', 'hg', 'svn'} function genOriginSaveResponse(objectType, saveRequestStatus, originUrl, saveRequestDate, saveTaskStatus) { return { - 'origin_type': objectType, + 'visit_type': objectType, 'save_request_status': saveRequestStatus, 'origin_url': originUrl, 'id': 1, diff --git a/docs/uri-scheme-browse-origin.rst b/docs/uri-scheme-browse-origin.rst --- a/docs/uri-scheme-browse-origin.rst +++ b/docs/uri-scheme-browse-origin.rst @@ -14,13 +14,11 @@ Origin visits """"""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visits/ +.. http:get:: /browse/origin/(origin_url)/visits/ HTML view that displays a visits reporting for a software origin identified by its type and url. - :param string origin_type: the type of software origin (possible values are ``git``, ``svn``, - ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive @@ -29,8 +27,8 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/visits/` - :swh_web_browse:`origin/git/url/https://github.com/python/cpython/visits/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/visits/` + :swh_web_browse:`origin/https://github.com/python/cpython/visits/` :swh_web_browse:`origin/deb://Debian-Security/packages/mediawiki/visits/` :swh_web_browse:`origin/https://gitorious.org/qt/qtbase.git/visits/` @@ -38,7 +36,7 @@ Origin directory """""""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/] +.. http:get:: /browse/origin/(origin_url)/directory/[(path)/] HTML view for browsing the content of a directory reachable from the root directory (including itself) associated to the latest full visit of a software origin. @@ -58,8 +56,6 @@ The origin branch (default to master) from which to retrieve the directory content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string path: optional parameter used to specify the path of a directory reachable from the origin root one @@ -79,14 +75,14 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/directory/` - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/directory/net/ethernet/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/directory/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/directory/net/ethernet/` :swh_web_browse:`origin/https://github.com/python/cpython/directory/` :swh_web_browse:`origin/https://github.com/python/cpython/directory/Python/` :swh_web_browse:`origin/https://github.com/python/cpython/directory/?branch=refs/heads/2.7` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/directory/[(path)/] +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/directory/[(path)/] HTML view for browsing the content of a directory reachable from the root directory (including itself) associated to a visit of a software @@ -107,8 +103,6 @@ The origin branch (default to master) from which to retrieve the directory content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. @@ -132,17 +126,17 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/visit/1493926809/directory/` - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/visit/2016-09-14T10:36:21/directory/net/ethernet/` - :swh_web_browse:`origin/git/url/https://github.com/python/cpython/visit/1474620651/directory/` - :swh_web_browse:`origin/git/url/https://github.com/python/cpython/visit/2017-05-05/directory/Python/` - :swh_web_browse:`origin/git/url/https://github.com/python/cpython/visit/2015-08/directory/?branch=refs/heads/2.7` + :swh_web_browse:`origin/https://github.com/torvalds/linux/visit/1493926809/directory/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/visit/2016-09-14T10:36:21/directory/net/ethernet/` + :swh_web_browse:`origin/https://github.com/python/cpython/visit/1474620651/directory/` + :swh_web_browse:`origin/https://github.com/python/cpython/visit/2017-05-05/directory/Python/` + :swh_web_browse:`origin/https://github.com/python/cpython/visit/2015-08/directory/?branch=refs/heads/2.7` Origin content """""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/content/(path)/ +.. http:get:: /browse/origin/(origin_url)/content/(path)/ HTML view that produces a display of a content associated to the latest full visit of a software origin. @@ -168,8 +162,6 @@ The origin branch (default to master) from which to retrieve the content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string path: path of a content reachable from the origin root directory :query string branch: specify the origin branch name from which @@ -188,12 +180,12 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/git/git/content/git.c/` - :swh_web_browse:`origin/git/url/https://github.com/git/git/content/git.c/` + :swh_web_browse:`origin/https://github.com/git/git/content/git.c/` + :swh_web_browse:`origin/https://github.com/git/git/content/git.c/` :swh_web_browse:`origin/https://github.com/mozilla/gecko-dev/content/js/src/json.cpp/` :swh_web_browse:`origin/https://github.com/git/git/content/git.c/?branch=refs/heads/next` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/content/(path)/ +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/content/(path)/ HTML view that produces a display of a content associated to a visit of a software origin closest to a provided timestamp. @@ -220,8 +212,6 @@ The origin branch (default to master) from which to retrieve the content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. @@ -243,8 +233,8 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/git/git/visit/1473933564/content/git.c/` - :swh_web_browse:`origin/git/url/https://github.com/git/git/visit/2016-05-05T00:0:00+00:00/content/git.c/` + :swh_web_browse:`origin/https://github.com/git/git/visit/1473933564/content/git.c/` + :swh_web_browse:`origin/https://github.com/git/git/visit/2016-05-05T00:0:00+00:00/content/git.c/` :swh_web_browse:`origin/https://github.com/mozilla/gecko-dev/visit/1490126182/content/js/src/json.cpp/` :swh_web_browse:`origin/https://github.com/mozilla/gecko-dev/visit/2017-03-21/content/js/src/json.cpp/#L904-L931` :swh_web_browse:`origin/https://github.com/git/git/visit/2017-09-15/content/git.c/?branch=refs/heads/next` @@ -253,7 +243,7 @@ Origin history """""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/log/ +.. http:get:: /browse/origin/(origin_url)/log/ HTML view that produces a display of revisions history heading to the last revision found during the latest visit of a software origin. @@ -285,8 +275,6 @@ The origin branch (default to master) from which to retrieve the content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :query int per_page: the number of log entries to display per page :query int offset: the number of revisions to skip before returning those to display @@ -307,12 +295,12 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/videolan/vlc/log/` + :swh_web_browse:`origin/https://github.com/videolan/vlc/log/` :swh_web_browse:`origin/https://github.com/Kitware/CMake/log/` :swh_web_browse:`origin/https://github.com/Kitware/CMake/log/?branch=refs/heads/release` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/log/ +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/log/ HTML view that produces a display of revisions history heading to the last revision found during a visit of a software origin closest @@ -342,8 +330,6 @@ The origin branch (default to master) from which to retrieve the content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. @@ -369,15 +355,15 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/videolan/vlc/visit/1459651262/log/` - :swh_web_browse:`origin/git/url/https://github.com/Kitware/CMake/visit/2016-04-01/log/` + :swh_web_browse:`origin/https://github.com/videolan/vlc/visit/1459651262/log/` + :swh_web_browse:`origin/https://github.com/Kitware/CMake/visit/2016-04-01/log/` :swh_web_browse:`origin/https://github.com/Kitware/CMake/visit/1438116814/log/?branch=refs/heads/release` :swh_web_browse:`origin/https://github.com/Kitware/CMake/visit/2017-05-05T03:14:23/log/?branch=refs/heads/release` Origin branches """"""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/branches/ +.. http:get:: /browse/origin/(origin_url)/branches/ HTML view that produces a display of the list of branches found during the latest full visit of a software origin. @@ -392,8 +378,6 @@ That list of branches is paginated, each page displaying a maximum of 100 branches. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive @@ -402,10 +386,10 @@ .. parsed-literal:: - :swh_web_browse:`origin/deb/url/deb://Debian/packages/linux/branches/` + :swh_web_browse:`origin/deb://Debian/packages/linux/branches/` :swh_web_browse:`origin/https://github.com/webpack/webpack/branches/` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/branches/ +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/branches/ HTML view that produces a display of the list of branches found during a visit of a software origin closest to the provided timestamp. @@ -420,8 +404,6 @@ That list of branches is paginated, each page displaying a maximum of 100 branches. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. @@ -432,13 +414,13 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/kripken/emscripten/visit/2017-05-05T12:02:03/branches/` + :swh_web_browse:`origin/https://github.com/kripken/emscripten/visit/2017-05-05T12:02:03/branches/` :swh_web_browse:`origin/deb://Debian/packages/apache2-mod-xforward/visit/2017-11-15T05:15:09/branches/` Origin releases """"""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/releases/ +.. http:get:: /browse/origin/(origin_url)/releases/ HTML view that produces a display of the list of releases found during the latest full visit of a software origin. @@ -453,8 +435,6 @@ That list of releases is paginated, each page displaying a maximum of 100 releases. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive @@ -463,10 +443,10 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/git/git/releases/` + :swh_web_browse:`origin/https://github.com/git/git/releases/` :swh_web_browse:`origin/https://github.com/webpack/webpack/releases/` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/releases/ +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/releases/ HTML view that produces a display of the list of releases found during a visit of a software origin closest to the provided timestamp. @@ -481,8 +461,6 @@ That list of releases is paginated, each page displaying a maximum of 100 releases. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. @@ -493,7 +471,7 @@ .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/visit/2017-11-21T19:37:42/releases/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/visit/2017-11-21T19:37:42/releases/` :swh_web_browse:`origin/https://github.com/Kitware/CMake/visit/2016-09-23T14:06:35/releases/` .. _highlightjs: https://highlightjs.org/ diff --git a/docs/uri-scheme-browse-revision.rst b/docs/uri-scheme-browse-revision.rst --- a/docs/uri-scheme-browse-revision.rst +++ b/docs/uri-scheme-browse-revision.rst @@ -18,9 +18,6 @@ :param string sha1_git: hexadecimal representation for the **sha1_git** identifier of a revision - :query string origin_type: used internally to associate a software origin type - (possible values are ``git``, ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, - ``ftp`` or ``deposit``) to the revision :query string origin_url: used internally to associate an origin url (e.g. https://github.com/user/repo) to the revision :query string timestamp: used internally to associate an origin visit to the diff --git a/docs/uri-scheme-browse-snapshot.rst b/docs/uri-scheme-browse-snapshot.rst --- a/docs/uri-scheme-browse-snapshot.rst +++ b/docs/uri-scheme-browse-snapshot.rst @@ -42,7 +42,7 @@ The features offered by the view are similar to the one for browsing a directory in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/]`). + (see :http:get:`/browse/origin/(origin_url)/directory/[(path)/]`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :param string path: optional parameter used to specify the path of a directory @@ -76,7 +76,7 @@ The features offered by the view are similar to the one for browsing a content in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/content/(path)/`). + (see :http:get:`/browse/origin/(origin_url)/content/(path)/`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :param string path: path of a content reachable from the snapshot root directory @@ -110,7 +110,7 @@ The features offered by the view are similar to the one for browsing the history in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/log/`). + (see :http:get:`/browse/origin/(origin_url)/log/`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :query string revs_breadcrumb: used internally to store @@ -148,7 +148,7 @@ The features offered by the view are similar to the one for browsing the list of branches in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/branches/`). + (see :http:get:`/browse/origin/(origin_url)/branches/`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :statuscode 200: no error @@ -172,7 +172,7 @@ The features offered by the view are similar to the one for browsing the list of releases in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/releases/`). + (see :http:get:`/browse/origin/(origin_url)/releases/`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :statuscode 200: no error diff --git a/docs/uri-scheme-browse.rst b/docs/uri-scheme-browse.rst --- a/docs/uri-scheme-browse.rst +++ b/docs/uri-scheme-browse.rst @@ -39,13 +39,13 @@ Wherever a revision context is expected in a path (i.e., a **/browse/revision/(sha1_git)/** path fragment) we can put in its stead a path fragment -of the form **/origin/(origin_type)/url/(origin_url)/[/visit/(timestamp)/][?branch=(branch)]**. +of the form **/origin/(origin_url)/[/visit/(timestamp)/][?branch=(branch)]**. Such a fragment is resolved, internally by the archive, to a revision **sha1_git** as follows: - if **timestamp** is absent: look for the most recent crawl of origin - identified by **origin_type** and **origin_url** + identified by **origin_url** - if **timestamp** is given: look for the closest crawl of origin identified - by **origin_type** and **origin_url** from timestamp **timestamp** + by **origin_url** from timestamp **timestamp** - if **branch** is given as a query parameter: look for the branch **branch** - if **branch** is absent: look for branch "HEAD" or "master" - return the revision **sha1_git** pointed by the chosen branch @@ -53,18 +53,15 @@ The already mentioned URLs for revision contexts can therefore be alternatively specified by users as: -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/]` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/directory/[(path)/]` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/content/(path)/` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/content/(path)/` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/log/` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/log/` +* :http:get:`/browse/origin/(origin_url)/directory/[(path)/]` +* :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/directory/[(path)/]` +* :http:get:`/browse/origin/(origin_url)/content/(path)/` +* :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/content/(path)/` +* :http:get:`/browse/origin/(origin_url)/log/` +* :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/log/` Typing: -- **origin_type** corresponds to the type of the archived origin: - ``git``, ``svn``,``hg``, ``deb``, ``pypi``, ``ftp`` or ``deposit`` - - **origin_url** corresponds to the URL the origin was crawled from, for instance https://github.com/(user)/(repo)/ diff --git a/swh/web/admin/origin_save.py b/swh/web/admin/origin_save.py --- a/swh/web/admin/origin_save.py +++ b/swh/web/admin/origin_save.py @@ -81,7 +81,7 @@ status=SAVE_REQUEST_PENDING) # create origin save tasks for previously pending requests for psr in pending_save_requests: - create_save_origin_request(psr.origin_type, psr.origin_url) + create_save_origin_request(psr.visit_type, psr.origin_url) status_code = 200 else: status_code = 400 @@ -149,29 +149,29 @@ return HttpResponse(status=status_code) -@admin_route(r'origin/save/request/accept/(?P.+)/url/(?P.+)/', # noqa +@admin_route(r'origin/save/request/accept/(?P.+)/url/(?P.+)/', # noqa view_name='admin-origin-save-request-accept') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) -def _admin_origin_save_request_accept(request, origin_type, origin_url): +def _admin_origin_save_request_accept(request, visit_type, origin_url): try: SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveAuthorizedOrigin.objects.create(url=origin_url) - create_save_origin_request(origin_type, origin_url) + create_save_origin_request(visit_type, origin_url) return HttpResponse(status=200) -@admin_route(r'origin/save/request/reject/(?P.+)/url/(?P.+)/', # noqa +@admin_route(r'origin/save/request/reject/(?P.+)/url/(?P.+)/', # noqa view_name='admin-origin-save-request-reject') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) -def _admin_origin_save_request_reject(request, origin_type, origin_url): +def _admin_origin_save_request_reject(request, visit_type, origin_url): try: SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveUnauthorizedOrigin.objects.create(url=origin_url) - sor = SaveOriginRequest.objects.get(origin_type=origin_type, + sor = SaveOriginRequest.objects.get(visit_type=visit_type, origin_url=origin_url, status=SAVE_REQUEST_PENDING) sor.status = SAVE_REQUEST_REJECTED diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -134,12 +134,10 @@ return response -@api_route(r'/origin/(?P[a-z]+)/url/(?P.+)/', - 'api-1-origin') @api_route(r'/origin/(?P.+)/get/', 'api-1-origin') @api_doc('/origin/') @format_docstring(return_origin=DOC_RETURN_ORIGIN) -def api_origin(request, origin_url, origin_type=None): +def api_origin(request, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/get/ @@ -161,48 +159,14 @@ .. parsed-literal:: - :swh_web_api:`origin/git/url/https://github.com/python/cpython/` + :swh_web_api:`origin/https://github.com/python/cpython/get/` - - .. http:get:: /api/1/origin/(origin_type)/url/(origin_url)/ - - Get information about a software origin. - - .. warning:: - - All endpoints using an ``origin_type`` are - deprecated and will be removed in the near future. Only those - using only an ``origin_url`` will remain available. - You should use :http:get:`/api/1/origin/(origin_url)/get/` instead. - - :param string origin_type: the origin type (possible values are - ``git``, ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or - ``deposit``) - :param string origin_url: the origin url - - {return_origin} - - {common_headers} - - **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, - :http:method:`options` - - :statuscode 200: no error - :statuscode 404: requested origin can not be found in the archive - - **Example:** - - .. parsed-literal:: - - :swh_web_api:`origin/git/url/https://github.com/python/cpython/` """ ori_dict = { - 'type': origin_type, 'url': origin_url } - ori_dict = {k: v for k, v in ori_dict.items() if ori_dict[k]} - error_msg = 'Origin %s not found.' % \ - (ori_dict.get('id') or ori_dict['url']) + + error_msg = 'Origin with url %s not found.' % ori_dict['url'] return api_lookup( service.lookup_origin, ori_dict, @@ -500,18 +464,16 @@ with_origin_visit_link=False)) -@api_route(r'/origin/(?P[a-z]+)/url/(?P.+)' +@api_route(r'/origin/(?P.+)' '/intrinsic-metadata', 'api-origin-intrinsic-metadata') @api_doc('/origin/intrinsic-metadata/') @format_docstring() -def api_origin_intrinsic_metadata(request, origin_type, origin_url): +def api_origin_intrinsic_metadata(request, origin_url): """ - .. http:get:: /api/1/origin/(origin_type)/url/(origin_url)/intrinsic-metadata + .. http:get:: /api/1/origin/(origin_url)/intrinsic-metadata Get intrinsic metadata of a software origin (as a JSON-LD/CodeMeta dictionary). - :param string origin_type: the origin type (possible values are ``git``, ``svn``, - ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the origin url :>json string ???: intrinsic metadata field of the origin @@ -528,14 +490,13 @@ .. parsed-literal:: - :swh_web_api:`origin/git/url/https://github.com/python/cpython/intrinsic-metadata` + :swh_web_api:`origin/https://github.com/python/cpython/intrinsic-metadata` """ # noqa ori_dict = { - 'type': origin_type, 'url': origin_url } - error_msg = 'Origin with URL %s not found' % ori_dict['url'] + error_msg = 'Origin with url %s not found' % ori_dict['url'] return api_lookup( service.lookup_origin_intrinsic_metadata, ori_dict, diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -12,16 +12,16 @@ ) -@api_route(r'/origin/save/(?P.+)/url/(?P.+)/', +@api_route(r'/origin/save/(?P.+)/url/(?P.+)/', 'api-1-save-origin', methods=['GET', 'POST'], throttle_scope='swh_save_origin') @never_cache @api_doc('/origin/save/') @format_docstring() -def api_save_origin(request, origin_type, origin_url): +def api_save_origin(request, visit_type, origin_url): """ - .. http:get:: /api/1/origin/save/(origin_type)/url/(origin_url)/ - .. http:post:: /api/1/origin/save/(origin_type)/url/(origin_url)/ + .. http:get:: /api/1/origin/save/(visit_type)/url/(origin_url)/ + .. http:post:: /api/1/origin/save/(visit_type)/url/(origin_url)/ Request the saving of a software origin into the archive or check the status of previously created save requests. @@ -52,14 +52,14 @@ request will return an array of objects (as multiple save requests might have been submitted for the same origin). - :param string origin_type: the type of origin to save + :param string visit_type: the type of visit to perform (currently the supported types are ``git``, ``hg`` and ``svn``) :param string origin_url: the url of the origin to save {common_headers} :>json string origin_url: the url of the origin to save - :>json string origin_type: the type of the origin to save + :>json string visit_type: the type of visit to perform :>json string save_request_date: the date (in iso format) the save request was issued :>json string save_request_status: the status of the save request, @@ -72,16 +72,17 @@ :http:method:`head`, :http:method:`options` :statuscode 200: no error - :statuscode 400: an invalid origin type or url has been provided + :statuscode 400: an invalid visit type or origin url has been provided :statuscode 403: the provided origin url is blacklisted :statuscode 404: no save requests have been found for a given origin """ if request.method == 'POST': - sor = create_save_origin_request(origin_type, origin_url) + sor = create_save_origin_request(visit_type, origin_url) del sor['id'] else: - sor = get_save_origin_requests(origin_type, origin_url) - for s in sor: del s['id'] # noqa + sor = get_save_origin_requests(visit_type, origin_url) + for s in sor: + del s['id'] return sor diff --git a/swh/web/assets/src/bundles/admin/origin-save.js b/swh/web/assets/src/bundles/admin/origin-save.js --- a/swh/web/assets/src/bundles/admin/origin-save.js +++ b/swh/web/assets/src/bundles/admin/origin-save.js @@ -73,8 +73,8 @@ } }, { - data: 'origin_type', - name: 'origin_type' + data: 'visit_type', + name: 'visit_type' }, { data: 'origin_url', @@ -295,7 +295,7 @@ if (selectedRow.length) { let acceptOriginSaveRequestCallback = () => { let rowData = selectedRow.data(); - let acceptSaveRequestUrl = Urls.admin_origin_save_request_accept(rowData['origin_type'], rowData['origin_url']); + let acceptSaveRequestUrl = Urls.admin_origin_save_request_accept(rowData['visit_type'], rowData['origin_url']); csrfPost(acceptSaveRequestUrl) .then(() => { pendingSaveRequestsTable.ajax.reload(null, false); @@ -314,7 +314,7 @@ if (selectedRow.length) { let rejectOriginSaveRequestCallback = () => { let rowData = selectedRow.data(); - let rejectSaveRequestUrl = Urls.admin_origin_save_request_reject(rowData['origin_type'], rowData['origin_url']); + let rejectSaveRequestUrl = Urls.admin_origin_save_request_reject(rowData['visit_type'], rowData['origin_url']); csrfPost(rejectSaveRequestUrl) .then(() => { pendingSaveRequestsTable.ajax.reload(null, false); diff --git a/swh/web/assets/src/bundles/browse/origin-search.js b/swh/web/assets/src/bundles/browse/origin-search.js --- a/swh/web/assets/src/bundles/browse/origin-search.js +++ b/swh/web/assets/src/bundles/browse/origin-search.js @@ -36,8 +36,8 @@ let origin = origins[i]; let browseUrl = Urls.browse_origin(origin.url); let tableRow = ``; - tableRow += `${origin.type}`; tableRow += `${encodeURI(origin.url)}`; + tableRow += ``; tableRow += ``; tableRow += ''; table.append(tableRow); @@ -47,6 +47,7 @@ fetch(latestSnapshotUrl) .then(response => response.json()) .then(data => { + $(`#visit-type-origin-${i}`).text(data.type); $(`#visit-status-origin-${i}`).children().remove(); if (data) { $(`#visit-status-origin-${i}`).append(''); diff --git a/swh/web/assets/src/bundles/save/index.js b/swh/web/assets/src/bundles/save/index.js --- a/swh/web/assets/src/bundles/save/index.js +++ b/swh/web/assets/src/bundles/save/index.js @@ -46,7 +46,7 @@ .then(response => response.json()) .then(data => { for (let originType of data) { - $('#swh-input-origin-type').append(``); + $('#swh-input-visit-type').append(``); } }); @@ -76,8 +76,8 @@ } }, { - data: 'origin_type', - name: 'origin_type' + data: 'visit_type', + name: 'visit_type' }, { data: 'origin_url', @@ -159,7 +159,7 @@ $('.alert').alert('close'); if (event.target.checkValidity()) { $(event.target).removeClass('was-validated'); - let originType = $('#swh-input-origin-type').val(); + let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); originSaveRequest(originType, originUrl, @@ -188,7 +188,7 @@ $('#swh-input-origin-url').on('input', function(event) { let originUrl = $(this).val().trim(); $(this).val(originUrl); - $('#swh-input-origin-type option').each(function() { + $('#swh-input-visit-type option').each(function() { let val = $(this).val(); if (val && originUrl.includes(val)) { $(this).prop('selected', true); @@ -213,7 +213,7 @@ } } }) === undefined; - let originType = $('#swh-input-origin-type').val(); + let originType = $('#swh-input-visit-type').val(); if (originType === 'git' && validUrl) { // additional checks for well known code hosting providers let githubIdx = originUrl.indexOf('://github.com'); @@ -276,7 +276,7 @@ event.preventDefault(); event.stopPropagation(); - let originType = $('#swh-input-origin-type').val(); + let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); originSaveRequest(originType, originUrl, diff --git a/swh/web/browse/utils.py b/swh/web/browse/utils.py --- a/swh/web/browse/utils.py +++ b/swh/web/browse/utils.py @@ -852,44 +852,7 @@ return revision_log_data -# list of origin types that can be found in the swh archive -# TODO: retrieve it dynamically in an efficient way instead -# of hardcoding it -_swh_origin_types = ['git', 'svn', 'deb', 'hg', 'ftp', 'deposit', - 'pypi', 'npm'] - - -def get_origin_info(origin_url, origin_type=None): - """ - Get info about a software origin. - Its main purpose is to automatically find an origin type - when it is not provided as parameter. - - Args: - origin_url (str): complete url of a software origin - origin_type (str): optional origin type - - Returns: - A dict with the following entries: - * type: the origin type - * url: the origin url - * id: the internal id of the origin - """ - if origin_type: - return service.lookup_origin({'type': origin_type, - 'url': origin_url}) - else: - for origin_type in _swh_origin_types: - try: - origin_info = service.lookup_origin({'type': origin_type, - 'url': origin_url}) - return origin_info - except Exception: - pass - raise NotFoundExc('Origin with url %s not found!' % escape(origin_url)) - - -def get_snapshot_context(snapshot_id=None, origin_type=None, origin_url=None, +def get_snapshot_context(snapshot_id=None, origin_url=None, timestamp=None, visit_id=None): """ Utility function to compute relevant information when navigating @@ -899,7 +862,6 @@ Args: snapshot_id (str): hexadecimal representation of a snapshot identifier, all other parameters will be ignored if it is provided - origin_type (str): the origin type (git, svn, deposit, ...) origin_url (str): the origin_url (e.g. https://github.com/(user)/(repo)/) timestamp (str): a datetime string for retrieving the closest @@ -939,7 +901,7 @@ swh_type = 'snapshot' if origin_url: swh_type = 'origin' - origin_info = get_origin_info(origin_url, origin_type) + origin_info = service.lookup_origin({'url': origin_url}) visit_info = get_origin_visit(origin_info, timestamp, visit_id, snapshot_id) @@ -961,8 +923,7 @@ get_origin_visit_snapshot(origin_info, timestamp, visit_id, snapshot_id) - url_args = {'origin_type': origin_type, - 'origin_url': origin_info['url']} + url_args = {'origin_url': origin_info['url']} query_params = {'visit_id': visit_id} @@ -1010,8 +971,6 @@ 'snapshot_size': snapshot_size, 'is_empty': is_empty, 'origin_info': origin_info, - # keep track if the origin type was provided as url argument - 'origin_type': origin_type, 'visit_info': visit_info, 'branches': branches, 'releases': releases, diff --git a/swh/web/browse/views/content.py b/swh/web/browse/views/content.py --- a/swh/web/browse/views/content.py +++ b/swh/web/browse/views/content.py @@ -179,7 +179,6 @@ checksum = hash_to_hex(checksum) content_data = request_content(query_string, raise_if_unavailable=False) - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) selected_language = request.GET.get('language', None) @@ -188,8 +187,7 @@ snapshot_context = None if origin_url: try: - snapshot_context = get_snapshot_context(None, origin_type, - origin_url) + snapshot_context = get_snapshot_context(origin_url=origin_url) except Exception: raw_cnt_url = reverse('browse-content', url_args={'query_string': query_string}) diff --git a/swh/web/browse/views/directory.py b/swh/web/browse/views/directory.py --- a/swh/web/browse/views/directory.py +++ b/swh/web/browse/views/directory.py @@ -41,15 +41,13 @@ sha1_git = dir_info['target'] dirs, files = get_directory_entries(sha1_git) - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: - snapshot_context = get_snapshot_context(None, origin_type, - origin_url) + snapshot_context = get_snapshot_context(origin_url=origin_url) except Exception: raw_dir_url = reverse('browse-directory', url_args={'sha1_git': sha1_git}) diff --git a/swh/web/browse/views/origin.py b/swh/web/browse/views/origin.py --- a/swh/web/browse/views/origin.py +++ b/swh/web/browse/views/origin.py @@ -16,9 +16,7 @@ reverse, format_utc_iso_date, parse_timestamp ) from swh.web.common.exc import handle_view_exception -from swh.web.browse.utils import ( - get_origin_info, get_snapshot_context -) +from swh.web.browse.utils import get_snapshot_context from swh.web.browse.browseurls import browse_route from .utils.snapshot_context import ( @@ -28,56 +26,43 @@ ) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/directory/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/directory/(?P.+)/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/directory/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/directory/(?P.+)/', - r'origin/(?P.+)/visit/(?P.+)/directory/', +@browse_route(r'origin/(?P.+)/visit/(?P.+)/directory/', r'origin/(?P.+)/visit/(?P.+)' '/directory/(?P.+)/', r'origin/(?P.+)/directory/', r'origin/(?P.+)/directory/(?P.+)/', view_name='browse-origin-directory') -def origin_directory_browse(request, origin_url, origin_type=None, +def origin_directory_browse(request, origin_url, timestamp=None, path=None): """Django view for browsing the content of a directory associated to an origin for a given visit. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/]` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/directory/[(path)/]` + * :http:get:`/browse/origin/(origin_url)/directory/[(path)/]` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/directory/[(path)/]` """ # noqa - return browse_snapshot_directory( - request, origin_type=origin_type, origin_url=origin_url, - timestamp=timestamp, path=path) + return browse_snapshot_directory(request, origin_url=origin_url, + timestamp=timestamp, path=path) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/content/(?P.+)/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/content/(?P.+)/', - r'origin/(?P.+)/visit/(?P.+)' +@browse_route(r'origin/(?P.+)/visit/(?P.+)' '/content/(?P.+)/', r'origin/(?P.+)/content/(?P.+)/', view_name='browse-origin-content') -def origin_content_browse(request, origin_url, origin_type=None, path=None, +def origin_content_browse(request, origin_url, path=None, timestamp=None): """Django view that produces an HTML display of a content associated to an origin for a given visit. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/content/(path)/` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/content/(path)/` + * :http:get:`/browse/origin/(origin_url)/content/(path)/` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/content/(path)/` """ # noqa language = request.GET.get('language', None) - return browse_snapshot_content(request, origin_type=origin_type, + return browse_snapshot_content(request, origin_url=origin_url, timestamp=timestamp, path=path, selected_language=language) @@ -85,84 +70,69 @@ PER_PAGE = 20 -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/log/', - r'origin/(?P[a-z]+)/url/(?P.+)/log/', - r'origin/(?P.+)/visit/(?P.+)/log/', +@browse_route(r'origin/(?P.+)/visit/(?P.+)/log/', r'origin/(?P.+)/log/', view_name='browse-origin-log') -def origin_log_browse(request, origin_url, origin_type=None, timestamp=None): +def origin_log_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of revisions history (aka the commit log) associated to a software origin. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/log/` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/log/` + * :http:get:`/browse/origin/(origin_url)/log/` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/log/` """ # noqa - return browse_snapshot_log(request, origin_type=origin_type, + return browse_snapshot_log(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/branches/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/branches/', - r'origin/(?P.+)/visit/(?P.+)/branches/', +@browse_route(r'origin/(?P.+)/visit/(?P.+)/branches/', r'origin/(?P.+)/branches/', view_name='browse-origin-branches') -def origin_branches_browse(request, origin_url, origin_type=None, - timestamp=None): +def origin_branches_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of the list of branches associated to an origin for a given visit. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/branches/` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/branches/` + * :http:get:`/browse/origin/(origin_url)/branches/` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/branches/` """ # noqa - return browse_snapshot_branches(request, origin_type=origin_type, + return browse_snapshot_branches(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/releases/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/releases/', - r'origin/(?P.+)/visit/(?P.+)/releases/', +@browse_route(r'origin/(?P.+)/visit/(?P.+)/releases/', r'origin/(?P.+)/releases/', view_name='browse-origin-releases') -def origin_releases_browse(request, origin_url, origin_type=None, - timestamp=None): +def origin_releases_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of the list of releases associated to an origin for a given visit. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/releases/` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/releases/` + * :http:get:`/browse/origin/(origin_url)/releases/` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/releases/` """ # noqa - return browse_snapshot_releases(request, origin_type=origin_type, + return browse_snapshot_releases(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visits/', - r'origin/(?P.+)/visits/', +@browse_route(r'origin/(?P.+)/visits/', view_name='browse-origin-visits') -def origin_visits_browse(request, origin_url, origin_type=None): +def origin_visits_browse(request, origin_url): """Django view that produces an HTML display of visits reporting for a swh origin identified by its id or its url. The url that points to it is - :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visits/`. + :http:get:`/browse/origin/(origin_url)/visits/`. """ try: - origin_info = get_origin_info(origin_url, origin_type) + origin_info = service.lookup_origin({'url': origin_url}) origin_visits = get_origin_visits(origin_info) - snapshot_context = get_snapshot_context(origin_type=origin_type, - origin_url=origin_url) + snapshot_context = get_snapshot_context(origin_url=origin_url) except Exception as exc: return handle_view_exception(request, exc) @@ -180,8 +150,7 @@ snapshot = visit['snapshot'] if visit['snapshot'] else '' visit['browse_url'] = reverse('browse-origin-directory', - url_args={'origin_type': origin_type, - 'origin_url': origin_url, + url_args={'origin_url': origin_url, 'timestamp': url_date}, query_params=query_params) if not snapshot: @@ -228,14 +197,12 @@ return HttpResponse(results, content_type='application/json') -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/', - r'origin/(?P.+)/', +@browse_route(r'origin/(?P.+)/', view_name='browse-origin') -def origin_browse(request, origin_url, origin_type=None): +def origin_browse(request, origin_url): """Django view that redirects to the display of the latest archived snapshot for a given software origin. """ last_snapshot_url = reverse('browse-origin-directory', - url_args={'origin_type': origin_type, - 'origin_url': origin_url}) + url_args={'origin_url': origin_url}) return redirect(last_snapshot_url) diff --git a/swh/web/browse/views/release.py b/swh/web/browse/views/release.py --- a/swh/web/browse/views/release.py +++ b/swh/web/browse/views/release.py @@ -33,7 +33,6 @@ snapshot_context = None origin_info = None snapshot_id = request.GET.get('snapshot_id', None) - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) @@ -41,10 +40,8 @@ visit_id = request.GET.get('visit_id', None) if origin_url: try: - snapshot_context = \ - get_snapshot_context(snapshot_id, origin_type, - origin_url, timestamp, - visit_id) + snapshot_context = get_snapshot_context( + snapshot_id, origin_url, timestamp, visit_id) except Exception: raw_rel_url = reverse('browse-release', url_args={'sha1_git': sha1_git}) @@ -142,7 +139,6 @@ if origin_info: release_data['context-independent release'] = \ gen_release_link(release['id']) - release_data['origin type'] = origin_info['type'] release_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) browse_snapshot_link = \ diff --git a/swh/web/browse/views/revision.py b/swh/web/browse/views/revision.py --- a/swh/web/browse/views/revision.py +++ b/swh/web/browse/views/revision.py @@ -145,16 +145,14 @@ try: revision = service.lookup_revision(sha1_git) snapshot_context = None - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) if origin_url: - snapshot_context = get_snapshot_context(None, origin_type, - origin_url, - timestamp, visit_id) + snapshot_context = get_snapshot_context( + origin_url=origin_url, timestamp=timestamp, visit_id=visit_id) except Exception as exc: return handle_view_exception(request, exc) @@ -269,7 +267,6 @@ revision = service.lookup_revision(sha1_git) origin_info = None snapshot_context = None - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) @@ -282,9 +279,9 @@ content_data = None if origin_url: try: - snapshot_context = get_snapshot_context(None, origin_type, - origin_url, - timestamp, visit_id) + snapshot_context = get_snapshot_context( + origin_url=origin_url, timestamp=timestamp, + visit_id=visit_id) except Exception: raw_rev_url = reverse('browse-revision', url_args={'sha1_git': sha1_git}) @@ -348,7 +345,6 @@ indent=4, separators=(',', ': '))) if origin_info: - revision_data['origin type'] = origin_info['type'] revision_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) revision_data['context-independent revision'] = \ @@ -376,7 +372,6 @@ path_info = gen_path_info(path) query_params = {'snapshot_id': snapshot_id, - 'origin_type': origin_type, 'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id} @@ -481,8 +476,7 @@ diff_revision_url = reverse('diff-revision', url_args={'sha1_git': sha1_git}, - query_params={'origin_type': origin_type, - 'origin': origin_url, + query_params={'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id}) diff --git a/swh/web/browse/views/snapshot.py b/swh/web/browse/views/snapshot.py --- a/swh/web/browse/views/snapshot.py +++ b/swh/web/browse/views/snapshot.py @@ -41,13 +41,11 @@ The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/directory/[(path)/]` """ - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) return browse_snapshot_directory(request, snapshot_id=snapshot_id, - path=path, origin_type=origin_type, - origin_url=origin_url) + path=path, origin_url=origin_url) @browse_route(r'snapshot/(?P[0-9a-f]+)/content/(?P.+)/', diff --git a/swh/web/browse/views/utils/snapshot_context.py b/swh/web/browse/views/utils/snapshot_context.py --- a/swh/web/browse/views/utils/snapshot_context.py +++ b/swh/web/browse/views/utils/snapshot_context.py @@ -93,30 +93,29 @@ msg = ('%s %s for snapshot with id %s' ' not found!' % (branch_type, branch, snapshot_id)) elif visit_id and not branches: - msg = ('Origin with type %s and url %s' + msg = ('Origin with url %s' ' for visit with id %s has an empty list' - ' of %s!' % (origin_info['type'], origin_info['url'], visit_id, + ' of %s!' % (origin_info['url'], visit_id, branch_type_plural)) elif visit_id: msg = ('%s %s associated to visit with' - ' id %s for origin with type %s and url %s' + ' id %s for origin with url %s' ' not found!' % (branch_type, branch, visit_id, - origin_info['type'], origin_info['url'])) + origin_info['url'])) elif not branches: - msg = ('Origin with type %s and url %s' + msg = ('Origin with url %s' ' for visit with timestamp %s has an empty list' - ' of %s!' % (origin_info['type'], origin_info['url'], + ' of %s!' % (origin_info['url'], timestamp, branch_type_plural)) else: msg = ('%s %s associated to visit with' - ' timestamp %s for origin with type %s' - ' and url %s not found!' % (branch_type, branch, timestamp, - origin_info['type'], - origin_info['url'])) + ' timestamp %s for origin with ' + 'url %s not found!' % (branch_type, branch, timestamp, + origin_info['url'])) raise NotFoundExc(escape(msg)) -def _process_snapshot_request(request, snapshot_id=None, origin_type=None, +def _process_snapshot_request(request, snapshot_id=None, origin_url=None, timestamp=None, path=None, browse_context='directory'): """ @@ -126,7 +125,7 @@ visit_id = request.GET.get('visit_id', None) - snapshot_context = get_snapshot_context(snapshot_id, origin_type, + snapshot_context = get_snapshot_context(snapshot_id, origin_url, timestamp, visit_id) swh_type = snapshot_context['swh_type'] @@ -221,7 +220,7 @@ return snapshot_context -def browse_snapshot_directory(request, snapshot_id=None, origin_type=None, +def browse_snapshot_directory(request, snapshot_id=None, origin_url=None, timestamp=None, path=None): """ Django view implementation for browsing a directory in a snapshot context. @@ -229,7 +228,7 @@ try: snapshot_context = _process_snapshot_request( - request, snapshot_id, origin_type, origin_url, + request, snapshot_id, origin_url, timestamp, path, browse_context='directory') root_sha1_git = snapshot_context['root_sha1_git'] @@ -339,10 +338,10 @@ 'context-independent snapshot': browse_snp_link} if origin_info: - dir_metadata['origin type'] = origin_info['type'] dir_metadata['origin url'] = origin_info['url'] dir_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) + dir_metadata['origin visit type'] = visit_info['type'] vault_cooking = { 'directory_context': True, @@ -393,7 +392,7 @@ 'swh_ids': swh_ids}) -def browse_snapshot_content(request, snapshot_id=None, origin_type=None, +def browse_snapshot_content(request, snapshot_id=None, origin_url=None, timestamp=None, path=None, selected_language=None): """ @@ -402,7 +401,7 @@ try: snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_type, origin_url, + origin_url, timestamp, path, browse_context='content') @@ -525,10 +524,10 @@ error_description = content_data['error_description'] if origin_info: - content_metadata['origin type'] = origin_info['type'] content_metadata['origin url'] = origin_info['url'] content_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) + content_metadata['origin visit type'] = visit_info['type'] browse_snapshot_link = gen_snapshot_link(snapshot_id) content_metadata['context-independent snapshot'] = browse_snapshot_link @@ -582,7 +581,7 @@ PER_PAGE = 100 -def browse_snapshot_log(request, snapshot_id=None, origin_type=None, +def browse_snapshot_log(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a revision history in a @@ -591,8 +590,7 @@ try: snapshot_context = _process_snapshot_request( - request, snapshot_id, origin_type, origin_url, - timestamp, browse_context='log') + request, snapshot_id, origin_url, timestamp, browse_context='log') revision_id = snapshot_context['revision_id'] @@ -670,10 +668,10 @@ } if origin_info: - revision_metadata['origin type'] = origin_info['type'] revision_metadata['origin url'] = origin_info['url'] revision_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) + revision_metadata['origin visit type'] = visit_info['type'] swh_objects = [{'type': 'revision', 'id': revision_id}, @@ -709,7 +707,7 @@ 'swh_ids': swh_ids}) -def browse_snapshot_branches(request, snapshot_id=None, origin_type=None, +def browse_snapshot_branches(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a list of branches in a snapshot @@ -717,8 +715,7 @@ """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_type, origin_url, - timestamp) + origin_url, timestamp) branches_bc = request.GET.get('branches_breadcrumbs', '') branches_bc = branches_bc.split(',') if branches_bc else [] @@ -748,8 +745,7 @@ else: revision_url = reverse('browse-revision', url_args={'sha1_git': branch['revision']}, - query_params={'origin_type': origin_type, - 'origin': origin_info['url']}) + query_params={'origin': origin_info['url']}) query_params['branch'] = branch['name'] directory_url = reverse(browse_view_name, url_args=url_args, @@ -799,7 +795,7 @@ 'snapshot_context': snapshot_context}) -def browse_snapshot_releases(request, snapshot_id=None, origin_type=None, +def browse_snapshot_releases(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a list of releases in a snapshot @@ -807,8 +803,7 @@ """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_type, origin_url, - timestamp) + origin_url, timestamp) rel_bc = request.GET.get('releases_breadcrumbs', '') rel_bc = rel_bc.split(',') if rel_bc else [] diff --git a/swh/web/common/migrations/0006_rename_origin_type.py b/swh/web/common/migrations/0006_rename_origin_type.py new file mode 100644 --- /dev/null +++ b/swh/web/common/migrations/0006_rename_origin_type.py @@ -0,0 +1,23 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('swh.web.common', '0005_remove_duplicated_authorized_origins'), + ] + + operations = [ + migrations.RenameField( + model_name='saveoriginrequest', + old_name='origin_type', + new_name='visit_type', + ), + ] diff --git a/swh/web/common/models.py b/swh/web/common/models.py --- a/swh/web/common/models.py +++ b/swh/web/common/models.py @@ -68,7 +68,7 @@ """ id = models.BigAutoField(primary_key=True) request_date = models.DateTimeField(auto_now_add=True) - origin_type = models.CharField(max_length=200, null=False) + visit_type = models.CharField(max_length=200, null=False) origin_url = models.CharField(max_length=200, null=False) status = models.TextField(choices=SAVE_REQUEST_STATUS, default=SAVE_REQUEST_PENDING) @@ -85,7 +85,7 @@ def __str__(self): return str({'id': self.id, 'request_date': self.request_date, - 'origin_type': self.origin_type, + 'visit_type': self.visit_type, 'origin_url': self.origin_url, 'status': self.status, 'loading_task_id': self.loading_task_id, diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -90,9 +90,9 @@ return SAVE_REQUEST_PENDING -# map origin type to scheduler task +# map visit type to scheduler task # TODO: do not hardcode the task name here (T1157) -_origin_type_task = { +_visit_type_task = { 'git': 'load-git', 'hg': 'load-hg', 'svn': 'load-svn' @@ -108,23 +108,23 @@ } -def get_savable_origin_types(): - return sorted(list(_origin_type_task.keys())) +def get_savable_visit_types(): + return sorted(list(_visit_type_task.keys())) -def _check_origin_type_savable(origin_type): +def _check_visit_type_savable(visit_type): """ - Get the list of software origin types that can be loaded + Get the list of visit types that can be performed through a save request. Returns: - list: the list of saveable origin types + list: the list of saveable visit types """ - allowed_origin_types = ', '.join(get_savable_origin_types()) - if origin_type not in _origin_type_task: - raise BadInputExc('Origin of type %s can not be saved! ' + allowed_visit_types = ', '.join(get_savable_visit_types()) + if visit_type not in _visit_type_task: + raise BadInputExc('Visit of type %s can not be saved! ' 'Allowed types are the following: %s' % - (origin_type, allowed_origin_types)) + (visit_type, allowed_visit_types)) _validate_url = URLValidator(schemes=['http', 'https', 'svn', 'git']) @@ -142,8 +142,7 @@ visit_date = None visit_status = None try: - origin = {'type': save_request.origin_type, - 'url': save_request.origin_url} + origin = {'url': save_request.origin_url} origin_info = service.lookup_origin(origin) origin_visits = get_origin_visits(origin_info) visit_dates = [parse_timestamp(v['date']) @@ -217,7 +216,7 @@ save_request.save() return {'id': save_request.id, - 'origin_type': save_request.origin_type, + 'visit_type': save_request.visit_type, 'origin_url': save_request.origin_url, 'save_request_date': save_request.request_date.isoformat(), 'save_request_status': save_request.status, @@ -225,14 +224,14 @@ 'visit_date': visit_date.isoformat() if visit_date else None} -def create_save_origin_request(origin_type, origin_url): +def create_save_origin_request(visit_type, origin_url): """ Create a loading task to save a software origin into the archive. This function aims to create a software origin loading task trough the use of the swh-scheduler component. - First, some checks are performed to see if the origin type and + First, some checks are performed to see if the visit type and origin url are valid but also if the the save request can be accepted. If those checks passed, the loading task is then created. Otherwise, the save request is put in pending or rejected state. @@ -241,18 +240,18 @@ database to keep track of them. Args: - origin_type (str): the type of origin to save (currently only + visit_type (str): the type of visit to perform (currently only ``git`` but ``svn`` and ``hg`` will soon be available) origin_url (str): the url of the origin to save Raises: - BadInputExc: the origin type or url is invalid + BadInputExc: the visit type or origin url is invalid ForbiddenExc: the provided origin url is blacklisted Returns: dict: A dict describing the save request with the following keys: - * **origin_type**: the type of the origin to save + * **visit_type**: the type of visit to perform * **origin_url**: the url of the origin * **save_request_date**: the date the request was submitted * **save_request_status**: the request status, either **accepted**, @@ -263,7 +262,7 @@ """ - _check_origin_type_savable(origin_type) + _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) save_request_status = can_save_origin(origin_url) task = None @@ -273,19 +272,19 @@ if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority kwargs = {'priority': 'high'} - # set task parameters according to the origin type - if origin_type == 'git': + # set task parameters according to the visit type + if visit_type == 'git': kwargs['repo_url'] = origin_url - elif origin_type == 'hg': + elif visit_type == 'hg': kwargs['origin_url'] = origin_url - elif origin_type == 'svn': + elif visit_type == 'svn': kwargs['origin_url'] = origin_url kwargs['svn_url'] = origin_url sor = None # get list of previously sumitted save requests current_sors = \ - list(SaveOriginRequest.objects.filter(origin_type=origin_type, + list(SaveOriginRequest.objects.filter(visit_type=visit_type, origin_url=origin_url)) can_create_task = False @@ -317,7 +316,7 @@ if can_create_task: # effectively create the scheduler task task_dict = create_oneshot_task_dict( - _origin_type_task[origin_type], **kwargs) + _visit_type_task[visit_type], **kwargs) task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted @@ -326,7 +325,7 @@ sor.loading_task_id = task['id'] sor.save() else: - sor = SaveOriginRequest.objects.create(origin_type=origin_type, + sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status, # noqa loading_task_id=task['id']) # noqa @@ -335,18 +334,18 @@ # check if there is already such a save request already submitted, # no need to add it to the database in that case try: - sor = SaveOriginRequest.objects.get(origin_type=origin_type, + sor = SaveOriginRequest.objects.get(visit_type=visit_type, origin_url=origin_url, status=save_request_status) # if not add it to the database except ObjectDoesNotExist: - sor = SaveOriginRequest.objects.create(origin_type=origin_type, + sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status) # origin can not be saved as its url is blacklisted, # log the request to the database anyway else: - sor = SaveOriginRequest.objects.create(origin_type=origin_type, + sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status) @@ -382,29 +381,30 @@ return save_requests -def get_save_origin_requests(origin_type, origin_url): +def get_save_origin_requests(visit_type, origin_url): """ Get all save requests for a given software origin. Args: - origin_type (str): the type of the origin + visit_type (str): the type of visit origin_url (str): the url of the origin Raises: - BadInputExc: the origin type or url is invalid + BadInputExc: the visit type or origin url is invalid NotFoundExc: no save requests can be found for the given origin Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ - _check_origin_type_savable(origin_type) + _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) - sors = SaveOriginRequest.objects.filter(origin_type=origin_type, + sors = SaveOriginRequest.objects.filter(visit_type=visit_type, origin_url=origin_url) if sors.count() == 0: - raise NotFoundExc(('No save requests found for origin with type ' - '%s and url %s.') % (origin_type, origin_url)) + raise NotFoundExc(('No save requests found for visit of type ' + '%s on origin with url %s.') + % (visit_type, origin_url)) return get_save_origin_requests_from_queryset(sors) diff --git a/swh/web/common/service.py b/swh/web/common/service.py --- a/swh/web/common/service.py +++ b/swh/web/common/service.py @@ -212,7 +212,7 @@ """Return information about the origin matching dict origin. Args: - origin: origin's dict with keys either 'id' or 'url' + origin: origin's dict with 'url' key Returns: origin information as dict. @@ -220,8 +220,7 @@ """ origin_info = storage.origin_get(origin) if not origin_info: - msg = 'Origin %s not found!' % \ - (origin.get('id') or origin['url']) + msg = 'Origin with url %s not found!' % origin['url'] raise NotFoundExc(msg) return converters.from_origin(origin_info) @@ -311,8 +310,7 @@ """ origin_info = storage.origin_get(origin_dict) if not origin_info: - msg = 'Origin with type %s and url %s not found!' % \ - (origin_dict['type'], origin_dict['url']) + msg = 'Origin with url %s not found!' % origin_dict['url'] raise NotFoundExc(msg) origins = [origin_info['url']] diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py --- a/swh/web/common/swh_templatetags.py +++ b/swh/web/common/swh_templatetags.py @@ -14,7 +14,7 @@ from docutils.writers.html4css1 import Writer, HTMLTranslator from inspect import cleandoc -from swh.web.common.origin_save import get_savable_origin_types +from swh.web.common.origin_save import get_savable_visit_types register = template.Library() @@ -158,17 +158,17 @@ @register.filter -def origin_type_savable(origin_type): +def visit_type_savable(visit_type): """Django template filter to check if a save request can be - created for a given origin type. + created for a given visit type. Args: - origin_type (str): the type of software origin + visit_type (str): the type of visit Returns: - If the origin type is saveable or not + If the visit type is saveable or not """ - return origin_type in get_savable_origin_types() + return visit_type in get_savable_visit_types() @register.filter diff --git a/swh/web/misc/origin_save.py b/swh/web/misc/origin_save.py --- a/swh/web/misc/origin_save.py +++ b/swh/web/misc/origin_save.py @@ -15,7 +15,7 @@ from swh.web.common.exc import ForbiddenExc from swh.web.common.models import SaveOriginRequest from swh.web.common.origin_save import ( - create_save_origin_request, get_savable_origin_types, + create_save_origin_request, get_savable_visit_types, get_save_origin_requests_from_queryset ) from swh.web.common.throttling import throttle_scope @@ -31,14 +31,14 @@ @api_view(['POST']) @authentication_classes((EnforceCSRFAuthentication, )) @throttle_scope('swh_save_origin') -def _origin_save_request(request, origin_type, origin_url): +def _origin_save_request(request, visit_type, origin_url): """ This view is called through AJAX from the save code now form of swh-web. We use DRF here as we want to rate limit the number of submitted requests per user to avoid being possibly flooded by bots. """ try: - response = json.dumps(create_save_origin_request(origin_type, + response = json.dumps(create_save_origin_request(visit_type, origin_url), separators=(',', ': ')) return HttpResponse(response, content_type='application/json') @@ -46,10 +46,10 @@ return HttpResponseForbidden(str(exc)) -def _origin_save_types_list(request): - origin_types = json.dumps(get_savable_origin_types(), - separators=(',', ': ')) - return HttpResponse(origin_types, content_type='application/json') +def _visit_save_types_list(request): + visit_types = json.dumps(get_savable_visit_types(), + separators=(',', ': ')) + return HttpResponse(visit_types, content_type='application/json') def _origin_save_requests_list(request, status): @@ -81,7 +81,7 @@ [sr for sr in save_requests if search_value.lower() in sr['save_request_status'].lower() or search_value.lower() in sr['save_task_status'].lower() - or search_value.lower() in sr['origin_type'].lower() + or search_value.lower() in sr['visit_type'].lower() or search_value.lower() in sr['origin_url'].lower()] table_data['recordsFiltered'] = len(save_requests) @@ -93,9 +93,9 @@ urlpatterns = [ url(r'^save/$', _origin_save_view, name='origin-save'), - url(r'^save/(?P.+)/url/(?P.+)/$', + url(r'^save/(?P.+)/url/(?P.+)/$', _origin_save_request, name='origin-save-request'), - url(r'^save/types/list/$', _origin_save_types_list, + url(r'^save/types/list/$', _visit_save_types_list, name='origin-save-types-list'), url(r'^save/requests/list/(?P.+)/$', _origin_save_requests_list, name='origin-save-requests-list'), diff --git a/swh/web/templates/browse/search.html b/swh/web/templates/browse/search.html --- a/swh/web/templates/browse/search.html +++ b/swh/web/templates/browse/search.html @@ -44,8 +44,8 @@ - + diff --git a/swh/web/templates/includes/take-new-snapshot.html b/swh/web/templates/includes/take-new-snapshot.html --- a/swh/web/templates/includes/take-new-snapshot.html +++ b/swh/web/templates/includes/take-new-snapshot.html @@ -8,7 +8,7 @@ {% load static %} {% load swh_templatetags %} -{% if snapshot_context and snapshot_context.origin_info and snapshot_context.origin_info.type|origin_type_savable %} +{% if snapshot_context and snapshot_context.visit_info and snapshot_context.visit_info.type|visit_type_savable %}
@@ -40,8 +40,8 @@ {% csrf_token %}
- - + +
diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html --- a/swh/web/templates/misc/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -34,7 +34,7 @@ save request. To do so, fill the required info in the form below:

Origin type Origin urlVisit type Visit status