diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js index 22718ac4..cd60bede 100644 --- a/cypress/integration/origin-save.spec.js +++ b/cypress/integration/origin-save.spec.js @@ -1,128 +1,128 @@ /** * Copyright (C) 2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ let url; let origin; const saveCodeMsg = { 'success': 'The "save code now" request has been accepted and will be processed as soon as possible.', 'warning': 'The "save code now" request has been put in pending state and may be accepted for processing after manual review.', 'rejected': 'The "save code now" request has been rejected because the provided origin url is blacklisted.', 'rateLimit': 'The rate limit for "save code now" requests has been reached. Please try again later.', 'unknownErr': 'An unexpected error happened when submitting the "save code now request' }; function makeOriginSaveRequest(originType, originUrl) { - cy.get('#swh-input-origin-type') + cy.get('#swh-input-visit-type') .select(originType) .get('#swh-input-origin-url') .type(originUrl) .get('#swh-save-origin-form') .submit(); } function checkAlertVisible(alertType, msg) { cy.get('#swh-origin-save-request-status') .should('be.visible') .find(`.alert-${alertType}`) .should('be.visible') .and('contain', msg); } // Stub requests to save an origin function stubSaveRequest(requestUrl, objectType, status, originUrl, taskStatus, responseStatus = 200) { cy.route({ method: 'POST', status: responseStatus, url: requestUrl, response: genOriginSaveResponse(objectType, status, originUrl, Date().toString(), taskStatus) }).as('saveRequest'); } // Mocks API response : /save/(:object_type)/(:origin_url) // object_type : {'git', 'hg', 'svn'} function genOriginSaveResponse(objectType, saveRequestStatus, originUrl, saveRequestDate, saveTaskStatus) { return { - 'origin_type': objectType, + 'visit_type': objectType, 'save_request_status': saveRequestStatus, 'origin_url': originUrl, 'id': 1, 'save_request_date': saveRequestDate, 'save_task_status': saveTaskStatus, 'visit_date': null }; }; describe('Origin Save Tests', function() { before(function() { url = this.Urls.origin_save(); origin = this.origin[0]; this.originSaveUrl = this.Urls.origin_save_request(origin.type, origin.url); }); beforeEach(function() { cy.visit(url); cy.server(); }); it('should display accepted message when accepted', function() { stubSaveRequest(this.originSaveUrl, origin.type, 'accepted', origin.url, 'not yet scheduled'); makeOriginSaveRequest(origin.type, origin.url); cy.wait('@saveRequest').then(() => { checkAlertVisible('success', saveCodeMsg['success']); }); }); it('should display warning message when pending', function() { stubSaveRequest(this.originSaveUrl, origin.type, 'pending', origin.url, 'not created'); makeOriginSaveRequest(origin.type, origin.url); cy.wait('@saveRequest').then(() => { checkAlertVisible('warning', saveCodeMsg['warning']); }); }); it('should show error when origin is rejected (status: 403)', function() { stubSaveRequest(this.originSaveUrl, origin.type, 'rejected', origin.url, 'not created', 403); makeOriginSaveRequest(origin.type, origin.url); cy.wait('@saveRequest').then(() => { checkAlertVisible('danger', saveCodeMsg['rejected']); }); }); it('should show error when rate limited (status: 429)', function() { stubSaveRequest(this.originSaveUrl, origin.type, 'Request was throttled. Expected available in 60 seconds.', origin.url, 'not created', 429); makeOriginSaveRequest(origin.type, origin.url); cy.wait('@saveRequest').then(() => { checkAlertVisible('danger', saveCodeMsg['rateLimit']); }); }); it('should show error when unknown error occurs (status other than 200, 403, 429)', function() { stubSaveRequest(this.originSaveUrl, origin.type, 'Error', origin.url, 'not created', 406); makeOriginSaveRequest(origin.type, origin.url); cy.wait('@saveRequest').then(() => { checkAlertVisible('danger', saveCodeMsg['unknownErr']); }); }); }); diff --git a/docs/uri-scheme-browse-origin.rst b/docs/uri-scheme-browse-origin.rst index 9f72b790..c76617c0 100644 --- a/docs/uri-scheme-browse-origin.rst +++ b/docs/uri-scheme-browse-origin.rst @@ -1,500 +1,478 @@ Origin ^^^^^^ This describes the URI scheme when one wants to browse the Software Heritage archive in the context of an origin (for instance, a repository crawled from GitHub or a Debian source package). All the views pointed by that scheme offer quick links to browse objects as found during the associated crawls performed by Software Heritage: * the root directory of the origin * the list of branches of the origin * the list of releases of the origin Origin visits """"""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visits/ +.. http:get:: /browse/origin/(origin_url)/visits/ HTML view that displays a visits reporting for a software origin identified by its type and url. - :param string origin_type: the type of software origin (possible values are ``git``, ``svn``, - ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/visits/` - :swh_web_browse:`origin/git/url/https://github.com/python/cpython/visits/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/visits/` + :swh_web_browse:`origin/https://github.com/python/cpython/visits/` :swh_web_browse:`origin/deb://Debian-Security/packages/mediawiki/visits/` :swh_web_browse:`origin/https://gitorious.org/qt/qtbase.git/visits/` Origin directory """""""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/] +.. http:get:: /browse/origin/(origin_url)/directory/[(path)/] HTML view for browsing the content of a directory reachable from the root directory (including itself) associated to the latest full visit of a software origin. The content of the directory is first sorted in lexicographical order and the sub-directories are displayed before the regular files. The view enables to navigate from the requested directory to directories reachable from it in a recursive way but also up to the origin root directory. A breadcrumb located in the top part of the view allows to keep track of the paths navigated so far. The view also enables to easily switch between the origin branches and releases through a dropdown menu. The origin branch (default to master) from which to retrieve the directory content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string path: optional parameter used to specify the path of a directory reachable from the origin root one :query string branch: specify the origin branch name from which to retrieve the root directory :query string release: specify the origin release name from which to retrieve the root directory :query string revision: specify the origin revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the root directory :query int visit_id: specify a visit id to retrieve the directory from instead of using the latest full visit by default :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive or the provided path does not exist from the origin root directory **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/directory/` - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/directory/net/ethernet/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/directory/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/directory/net/ethernet/` :swh_web_browse:`origin/https://github.com/python/cpython/directory/` :swh_web_browse:`origin/https://github.com/python/cpython/directory/Python/` :swh_web_browse:`origin/https://github.com/python/cpython/directory/?branch=refs/heads/2.7` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/directory/[(path)/] +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/directory/[(path)/] HTML view for browsing the content of a directory reachable from the root directory (including itself) associated to a visit of a software origin closest to a provided timestamp. The content of the directory is first sorted in lexicographical order and the sub-directories are displayed before the regular files. The view enables to navigate from the requested directory to directories reachable from it in a recursive way but also up to the origin root directory. A breadcrumb located in the top part of the view allows to keep track of the paths navigated so far. The view also enables to easily switch between the origin branches and releases through a dropdown menu. The origin branch (default to master) from which to retrieve the directory content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. :param path: optional parameter used to specify the path of a directory reachable from the origin root one :type path: string :query string branch: specify the origin branch name from which to retrieve the root directory :query string release: specify the origin release name from which to retrieve the root directory :query string revision: specify the origin revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the directory :query int visit_id: specify a visit id to retrieve the directory from instead of using the provided timestamp :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive, requested visit timestamp does not exist or the provided path does not exist from the origin root directory **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/visit/1493926809/directory/` - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/visit/2016-09-14T10:36:21/directory/net/ethernet/` - :swh_web_browse:`origin/git/url/https://github.com/python/cpython/visit/1474620651/directory/` - :swh_web_browse:`origin/git/url/https://github.com/python/cpython/visit/2017-05-05/directory/Python/` - :swh_web_browse:`origin/git/url/https://github.com/python/cpython/visit/2015-08/directory/?branch=refs/heads/2.7` + :swh_web_browse:`origin/https://github.com/torvalds/linux/visit/1493926809/directory/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/visit/2016-09-14T10:36:21/directory/net/ethernet/` + :swh_web_browse:`origin/https://github.com/python/cpython/visit/1474620651/directory/` + :swh_web_browse:`origin/https://github.com/python/cpython/visit/2017-05-05/directory/Python/` + :swh_web_browse:`origin/https://github.com/python/cpython/visit/2015-08/directory/?branch=refs/heads/2.7` Origin content """""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/content/(path)/ +.. http:get:: /browse/origin/(origin_url)/content/(path)/ HTML view that produces a display of a content associated to the latest full visit of a software origin. If the content to display is textual, it will be highlighted client-side if possible using highlightjs_. The procedure to perform that task is described in :http:get:`/browse/content/[(algo_hash):](hash)/`. It is also possible to highlight specific lines of a textual content (not in terms of syntax highlighting but to emphasize some relevant content part) by either: * clicking on line numbers (holding shift to highlight a lines range) * using an url fragment in the form '#Ln' or '#Lm-Ln' The view displays a breadcrumb on top of the rendered content in order to easily navigate up to the origin root directory. The view also enables to easily switch between the origin branches and releases through a dropdown menu. The origin branch (default to master) from which to retrieve the content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string path: path of a content reachable from the origin root directory :query string branch: specify the origin branch name from which to retrieve the content :query string release: specify the origin release name from which to retrieve the content :query string revision: specify the origin revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the content :query int visit_id: specify a visit id to retrieve the content from instead of using the latest full visit by default :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive, or the provided content path does not exist from the origin root directory **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/git/git/content/git.c/` - :swh_web_browse:`origin/git/url/https://github.com/git/git/content/git.c/` + :swh_web_browse:`origin/https://github.com/git/git/content/git.c/` + :swh_web_browse:`origin/https://github.com/git/git/content/git.c/` :swh_web_browse:`origin/https://github.com/mozilla/gecko-dev/content/js/src/json.cpp/` :swh_web_browse:`origin/https://github.com/git/git/content/git.c/?branch=refs/heads/next` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/content/(path)/ +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/content/(path)/ HTML view that produces a display of a content associated to a visit of a software origin closest to a provided timestamp. If the content to display is textual, it will be highlighted client-side if possible using highlightjs_. The procedure to perform that task is described in :http:get:`/browse/content/[(algo_hash):](hash)/`. It is also possible to highlight specific lines of a textual content (not in terms of syntax highlighting but to emphasize some relevant content part) by either: * clicking on line numbers (holding shift to highlight a lines range) * using an url fragment in the form '#Ln' or '#Lm-Ln' The view displays a breadcrumb on top of the rendered content in order to easily navigate up to the origin root directory. The view also enables to easily switch between the origin branches and releases through a dropdown menu. The origin branch (default to master) from which to retrieve the content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. :param string path: path of a content reachable from the origin root directory :query string branch: specify the origin branch name from which to retrieve the content :query string release: specify the origin release name from which to retrieve the content :query string revision: specify the origin revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the content :query int visit_id: specify a visit id to retrieve the content from instead of using the provided timestamp :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive, requested visit timestamp does not exist or the provided content path does not exist from the origin root directory **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/git/git/visit/1473933564/content/git.c/` - :swh_web_browse:`origin/git/url/https://github.com/git/git/visit/2016-05-05T00:0:00+00:00/content/git.c/` + :swh_web_browse:`origin/https://github.com/git/git/visit/1473933564/content/git.c/` + :swh_web_browse:`origin/https://github.com/git/git/visit/2016-05-05T00:0:00+00:00/content/git.c/` :swh_web_browse:`origin/https://github.com/mozilla/gecko-dev/visit/1490126182/content/js/src/json.cpp/` :swh_web_browse:`origin/https://github.com/mozilla/gecko-dev/visit/2017-03-21/content/js/src/json.cpp/#L904-L931` :swh_web_browse:`origin/https://github.com/git/git/visit/2017-09-15/content/git.c/?branch=refs/heads/next` Origin history """""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/log/ +.. http:get:: /browse/origin/(origin_url)/log/ HTML view that produces a display of revisions history heading to the last revision found during the latest visit of a software origin. In other words, it shows the commit log associated to the latest full visit of a software origin. The following data are displayed for each log entry: * link to browse the associated revision in the origin context * author of the revision * date of the revision * message associated the revision * commit date of the revision By default, the revisions are ordered in reverse chronological order of their commit date. N log entries are displayed per page (default is 100). In order to navigate in a large history, two buttons are present at the bottom of the view: * **Newer**: fetch and display if available the N more recent log entries than the ones currently displayed * **Older**: fetch and display if available the N older log entries than the ones currently displayed The view also enables to easily switch between the origin branches and releases through a dropdown menu. The origin branch (default to master) from which to retrieve the content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :query int per_page: the number of log entries to display per page :query int offset: the number of revisions to skip before returning those to display :query str revs_ordering: specify the revisions ordering, possible values are ``committer_date``, ``dfs``, ``dfs_post`` and ``bfs`` :query string branch: specify the origin branch name from which to retrieve the commit log :query string release: specify the origin release name from which to retrieve the commit log :query string revision: specify the origin revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the commit log :query int visit_id: specify a visit id to retrieve the history log from instead of using the latest visit by default :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/videolan/vlc/log/` + :swh_web_browse:`origin/https://github.com/videolan/vlc/log/` :swh_web_browse:`origin/https://github.com/Kitware/CMake/log/` :swh_web_browse:`origin/https://github.com/Kitware/CMake/log/?branch=refs/heads/release` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/log/ +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/log/ HTML view that produces a display of revisions history heading to the last revision found during a visit of a software origin closest to the provided timestamp. In other words, it shows the commit log associated to a visit of a software origin closest to a provided timestamp. The following data are displayed for each log entry: * author of the revision * link to the revision metadata * message associated the revision * date of the revision * link to browse the associated source tree in the origin context N log entries are displayed per page (default is 20). In order to navigate in a large history, two buttons are present at the bottom of the view: * **Newer**: fetch and display if available the N more recent log entries than the ones currently displayed * **Older**: fetch and display if available the N older log entries than the ones currently displayed The view also enables to easily switch between the origin branches and releases through a dropdown menu. The origin branch (default to master) from which to retrieve the content can also be specified by using the branch query parameter. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. :query string revs_breadcrumb: used internally to store the navigation breadcrumbs (i.e. the list of descendant revisions visited so far). It must be a string in the form "(rev_1)[/(rev_2)/.../(rev_n)]" where rev_i corresponds to a revision **sha1_git**. :query int per_page: the number of log entries to display per page (default is 20, max is 50) :query string branch: specify the origin branch name from which to retrieve the commit log :query string release: specify the origin release name from which to retrieve the commit log :query string revision: specify the origin revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the commit log :query int visit_id: specify a visit id to retrieve the history log from instead of using the provided timestamp :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/videolan/vlc/visit/1459651262/log/` - :swh_web_browse:`origin/git/url/https://github.com/Kitware/CMake/visit/2016-04-01/log/` + :swh_web_browse:`origin/https://github.com/videolan/vlc/visit/1459651262/log/` + :swh_web_browse:`origin/https://github.com/Kitware/CMake/visit/2016-04-01/log/` :swh_web_browse:`origin/https://github.com/Kitware/CMake/visit/1438116814/log/?branch=refs/heads/release` :swh_web_browse:`origin/https://github.com/Kitware/CMake/visit/2017-05-05T03:14:23/log/?branch=refs/heads/release` Origin branches """"""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/branches/ +.. http:get:: /browse/origin/(origin_url)/branches/ HTML view that produces a display of the list of branches found during the latest full visit of a software origin. The following data are displayed for each branch: * its name * a link to browse the associated directory * a link to browse the associated revision * last commit message * last commit date That list of branches is paginated, each page displaying a maximum of 100 branches. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/deb/url/deb://Debian/packages/linux/branches/` + :swh_web_browse:`origin/deb://Debian/packages/linux/branches/` :swh_web_browse:`origin/https://github.com/webpack/webpack/branches/` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/branches/ +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/branches/ HTML view that produces a display of the list of branches found during a visit of a software origin closest to the provided timestamp. The following data are displayed for each branch: * its name * a link to browse the associated directory * a link to browse the associated revision * last commit message * last commit date That list of branches is paginated, each page displaying a maximum of 100 branches. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/kripken/emscripten/visit/2017-05-05T12:02:03/branches/` + :swh_web_browse:`origin/https://github.com/kripken/emscripten/visit/2017-05-05T12:02:03/branches/` :swh_web_browse:`origin/deb://Debian/packages/apache2-mod-xforward/visit/2017-11-15T05:15:09/branches/` Origin releases """"""""""""""" -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/releases/ +.. http:get:: /browse/origin/(origin_url)/releases/ HTML view that produces a display of the list of releases found during the latest full visit of a software origin. The following data are displayed for each release: * its name * a link to browse the release details * its target type (revision, directory, content or release) * its associated message * its date That list of releases is paginated, each page displaying a maximum of 100 releases. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/git/git/releases/` + :swh_web_browse:`origin/https://github.com/git/git/releases/` :swh_web_browse:`origin/https://github.com/webpack/webpack/releases/` -.. http:get:: /browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/releases/ +.. http:get:: /browse/origin/(origin_url)/visit/(timestamp)/releases/ HTML view that produces a display of the list of releases found during a visit of a software origin closest to the provided timestamp. The following data are displayed for each release: * its name * a link to browse the release details * its target type (revision, directory, content or release) * its associated message * its date That list of releases is paginated, each page displaying a maximum of 100 releases. - :param string origin_type: the type of software origin (possible values are ``git``, - ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the url of the origin (e.g. https://github.com/(user)/(repo)/) :param string timestamp: a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Examples:** .. parsed-literal:: - :swh_web_browse:`origin/git/url/https://github.com/torvalds/linux/visit/2017-11-21T19:37:42/releases/` + :swh_web_browse:`origin/https://github.com/torvalds/linux/visit/2017-11-21T19:37:42/releases/` :swh_web_browse:`origin/https://github.com/Kitware/CMake/visit/2016-09-23T14:06:35/releases/` .. _highlightjs: https://highlightjs.org/ .. _dateutil.parser.parse: http://dateutil.readthedocs.io/en/stable/parser.html diff --git a/docs/uri-scheme-browse-revision.rst b/docs/uri-scheme-browse-revision.rst index 845aeb7c..79a65961 100644 --- a/docs/uri-scheme-browse-revision.rst +++ b/docs/uri-scheme-browse-revision.rst @@ -1,82 +1,79 @@ Revision ^^^^^^^^ .. http:get:: /browse/revision/(sha1_git)/ HTML view to browse a revision. It notably shows the revision date and message but also offers links to get more details on: * its author * its parent revisions * the history log reachable from it The view also enables to navigate in the source tree associated to the revision and browse its content. Last but not least, the view displays the list of file changes introduced in the revision but also the diffs of each changed files. :param string sha1_git: hexadecimal representation for the **sha1_git** identifier of a revision - :query string origin_type: used internally to associate a software origin type - (possible values are ``git``, ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, - ``ftp`` or ``deposit``) to the revision :query string origin_url: used internally to associate an origin url (e.g. https://github.com/user/repo) to the revision :query string timestamp: used internally to associate an origin visit to the revision, must be a date string (any format parsable by `dateutil.parser.parse`_) or Unix timestamp to parse in order to find the closest visit. :query int visit_id: used internally to specify a visit id instead of using the provided timestamp :query string path: used internally when navigating in the source tree associated to the revision :statuscode 200: no error :statuscode 404: requested revision can not be found in the archive **Examples:** .. parsed-literal:: :swh_web_browse:`revision/f1b94134a4b879bc55c3dacdb496690c8ebdc03f/` :swh_web_browse:`revision/d1aa2b3f607b35dc5dbf613b2334b6d243ec2bda/` .. _dateutil.parser.parse: http://dateutil.readthedocs.io/en/stable/parser.html .. http:get:: /browse/revision/(sha1_git)/log/ HTML view that displays the list of revisions heading to a given one. In other words, it shows a commit log. The following data are displayed for each log entry: * link to browse the revision * author of the revision * date of the revision * message associated to the revision * commit date of the revision By default, the revisions are ordered in reverse chronological order of their commit date. N log entries are displayed per page (default is 100). In order to navigate in a large history, two buttons are present at the bottom of the view: * **Newer**: fetch and display if available the N more recent log entries than the ones currently displayed * **Older**: fetch and display if available the N older log entries than the ones currently displayed :param string sha1_git: hexadecimal representation for the **sha1_git** identifier of a revision :query int per_page: the number of log entries to display per page :query int offset: the number of revisions to skip before returning those to display :query str revs_ordering: specify the revisions ordering, possible values are ``committer_date``, ``dfs``, ``dfs_post`` and ``bfs`` :statuscode 200: no error :statuscode 404: requested revision can not be found in the archive **Examples:** .. parsed-literal:: :swh_web_browse:`revision/f1b94134a4b879bc55c3dacdb496690c8ebdc03f/log/` :swh_web_browse:`revision/d1aa2b3f607b35dc5dbf613b2334b6d243ec2bda/log/` diff --git a/docs/uri-scheme-browse-snapshot.rst b/docs/uri-scheme-browse-snapshot.rst index 8f5f2fd7..4f196fa8 100644 --- a/docs/uri-scheme-browse-snapshot.rst +++ b/docs/uri-scheme-browse-snapshot.rst @@ -1,187 +1,187 @@ Snapshot ^^^^^^^^ .. http:get:: /browse/snapshot/(snapshot_id)/ HTML view that displays the content of a snapshot from its identifier (see :func:`swh.model.identifiers.snapshot_identifier` in our data model module for details about how they are computed). A snapshot is a set of named branches, which are pointers to objects at any level of the Software Heritage DAG. It represents a full picture of an origin at a given time. Thus, multiple visits of different origins can point to the same snapshot (for instance, when several projects are forks of a common one). Currently, that endpoint simply performs a redirection to :http:get:`/browse/snapshot/(snapshot_id)/directory/[(path)/]` in order to display the root directory associated to the default snapshot branch (usually master). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :statuscode 200: no error :statuscode 400: an invalid snapshot identifier has been provided :statuscode 404: requested snapshot can not be found in the archive **Examples:** .. parsed-literal:: :swh_web_browse:`snapshot/baebc2109e4a2ec22a1129a3859647e191d04df4/` :swh_web_browse:`snapshot/673156c31a876c5b99b2fe3e89615529de9a3c44/` Snapshot directory """""""""""""""""" .. http:get:: /browse/snapshot/(snapshot_id)/directory/[(path)/] HTML view that displays the content of a directory reachable from a snapshot. The features offered by the view are similar to the one for browsing a directory in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/]`). + (see :http:get:`/browse/origin/(origin_url)/directory/[(path)/]`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :param string path: optional parameter used to specify the path of a directory reachable from the snapshot root one :query string branch: specify the snapshot branch name from which to retrieve the root directory :query string release: specify the snapshot release name from which to retrieve the root directory :query string revision: specify the snapshot revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the root directory :statuscode 200: no error :statuscode 400: an invalid snapshot identifier has been provided :statuscode 404: requested snapshot can not be found in the archive **Examples:** .. parsed-literal:: :swh_web_browse:`snapshot/baebc2109e4a2ec22a1129a3859647e191d04df4/directory/drivers/gpu/` :swh_web_browse:`snapshot/673156c31a876c5b99b2fe3e89615529de9a3c44/directory/src/opengl/` :swh_web_browse:`snapshot/673156c31a876c5b99b2fe3e89615529de9a3c44/log/?release=v5.7.0` Snapshot content """""""""""""""" .. http:get:: /browse/snapshot/(snapshot_id)/content/(path)/ HTML view that produces a display of a content reachable from a snapshot. The features offered by the view are similar to the one for browsing a content in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/content/(path)/`). + (see :http:get:`/browse/origin/(origin_url)/content/(path)/`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :param string path: path of a content reachable from the snapshot root directory :query string branch: specify the snapshot branch name from which to retrieve the content :query string release: specify the snapshot release name from which to retrieve the content :query string revision: specify the snapshot revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the content :statuscode 200: no error :statuscode 400: an invalid snapshot identifier has been provided :statuscode 404: requested snapshot can not be found in the archive, or the provided content path does not exist from the origin root directory **Examples:** .. parsed-literal:: :swh_web_browse:`snapshot/baebc2109e4a2ec22a1129a3859647e191d04df4/content/init/initramfs.c` :swh_web_browse:`snapshot/673156c31a876c5b99b2fe3e89615529de9a3c44/content/src/opengl/qglbuffer.h/` :swh_web_browse:`snapshot/673156c31a876c5b99b2fe3e89615529de9a3c44/content/src/opengl/qglbuffer.h/?release=v5.0.0` Snapshot history """""""""""""""" .. http:get:: /browse/snapshot/(snapshot_id)/log/ HTML view that produces a display of revisions history (aka the commit log) heading to the last revision collected in a snapshot. The features offered by the view are similar to the one for browsing the history in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/log/`). + (see :http:get:`/browse/origin/(origin_url)/log/`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :query string revs_breadcrumb: used internally to store the navigation breadcrumbs (i.e. the list of descendant revisions visited so far). It must be a string in the form "(rev_1)[/(rev_2)/.../(rev_n)]" where rev_i corresponds to a revision **sha1_git**. :query int per_page: the number of log entries to display per page (default is 20, max is 50) :query string branch: specify the snapshot branch name from which to retrieve the commit log :query string release: specify the snapshot release name from which to retrieve the commit log :query string revision: specify the snapshot revision, identified by the hexadecimal representation of its **sha1_git** value, from which to retrieve the commit log :statuscode 200: no error :statuscode 400: an invalid snapshot identifier has been provided :statuscode 404: requested snapshot can not be found in the archive **Examples:** .. parsed-literal:: :swh_web_browse:`snapshot/a274b44111f777209556e94920b7e71cf5c305cd/log/` :swh_web_browse:`snapshot/9ca9e75279df5f4e3fee19bf5190ed672dcdfb33/log/?branch=refs/heads/emacs-unicode` Snapshot branches """"""""""""""""" .. http:get:: /browse/snapshot/(snapshot_id)/branches/ HTML view that produces a display of the list of branches collected in a snapshot. The features offered by the view are similar to the one for browsing the list of branches in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/branches/`). + (see :http:get:`/browse/origin/(origin_url)/branches/`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :statuscode 200: no error :statuscode 400: an invalid snapshot identifier has been provided :statuscode 404: requested snapshot can not be found in the archive **Examples:** .. parsed-literal:: :swh_web_browse:`snapshot/03d7897352541e78ee7b13a580dc836778e8126a/branches/` :swh_web_browse:`snapshot/f37563b953327f8fd83e39af6ebb929ef85103d5/branches/` Snapshot releases """"""""""""""""" .. http:get:: /browse/snapshot/(snapshot_id)/releases/ HTML view that produces a display of the list of releases collected in a snapshot. The features offered by the view are similar to the one for browsing the list of releases in an origin context - (see :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/releases/`). + (see :http:get:`/browse/origin/(origin_url)/releases/`). :param string snapshot_id: hexadecimal representation of the snapshot **sha1** identifier :statuscode 200: no error :statuscode 400: an invalid snapshot identifier has been provided :statuscode 404: requested snapshot can not be found in the archive **Examples:** .. parsed-literal:: :swh_web_browse:`snapshot/673156c31a876c5b99b2fe3e89615529de9a3c44/releases/` :swh_web_browse:`snapshot/23e6fb084a60cc909b9e222d80d89fdb98756dee/releases/` diff --git a/docs/uri-scheme-browse.rst b/docs/uri-scheme-browse.rst index c04cc241..fc9499f0 100644 --- a/docs/uri-scheme-browse.rst +++ b/docs/uri-scheme-browse.rst @@ -1,99 +1,96 @@ URI scheme for swh-web Browse application ========================================= This web application aims to provide HTML views to easily navigate in the archive, thus it needs to be reached from a web browser. If you intend to query the archive programmatically through any HTTP client, please refer to the :ref:`swh-web-api-urls` section instead. Context-independent browsing ---------------------------- Context-independent URLs provide information about objects (e.g., revisions, directories, contents, person, ...), independently of the contexts where they have been found (e.g., specific repositories, branches, commits, ...). The following endpoints are the same of the API case (see below), and just render the corresponding information for user consumption. Where hyperlinks are created, they always point to other context-independent user URLs: * :http:get:`/browse/content/[(algo_hash):](hash)/`: Display a content * :http:get:`/browse/content/[(algo_hash):](hash)/raw/`: Get / Download content raw data * :http:get:`/browse/directory/(sha1_git)/[(path)/]`: Browse the content of a directory * :http:get:`/browse/person/(person_id)/`: Information on a person * :http:get:`/browse/revision/(sha1_git)/`: Browse a revision * :http:get:`/browse/revision/(sha1_git)/log/`: Browse history log heading to a revision Context-dependent browsing -------------------------- Context-dependent URLs provide information about objects, limited to specific contexts where the objects have been found. For instance, instead of having to specify a (root) revision by **sha1_git**, users might want to specify a place and a time. In Software Heritage a "place" is an origin, with an optional branch name; a "time" is a timestamp at which some place has been observed by Software Heritage crawlers. Wherever a revision context is expected in a path (i.e., a **/browse/revision/(sha1_git)/** path fragment) we can put in its stead a path fragment -of the form **/origin/(origin_type)/url/(origin_url)/[/visit/(timestamp)/][?branch=(branch)]**. +of the form **/origin/(origin_url)/[/visit/(timestamp)/][?branch=(branch)]**. Such a fragment is resolved, internally by the archive, to a revision **sha1_git** as follows: - if **timestamp** is absent: look for the most recent crawl of origin - identified by **origin_type** and **origin_url** + identified by **origin_url** - if **timestamp** is given: look for the closest crawl of origin identified - by **origin_type** and **origin_url** from timestamp **timestamp** + by **origin_url** from timestamp **timestamp** - if **branch** is given as a query parameter: look for the branch **branch** - if **branch** is absent: look for branch "HEAD" or "master" - return the revision **sha1_git** pointed by the chosen branch The already mentioned URLs for revision contexts can therefore be alternatively specified by users as: -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/]` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/directory/[(path)/]` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/content/(path)/` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/content/(path)/` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/log/` -* :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/log/` +* :http:get:`/browse/origin/(origin_url)/directory/[(path)/]` +* :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/directory/[(path)/]` +* :http:get:`/browse/origin/(origin_url)/content/(path)/` +* :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/content/(path)/` +* :http:get:`/browse/origin/(origin_url)/log/` +* :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/log/` Typing: -- **origin_type** corresponds to the type of the archived origin: - ``git``, ``svn``,``hg``, ``deb``, ``pypi``, ``ftp`` or ``deposit`` - - **origin_url** corresponds to the URL the origin was crawled from, for instance https://github.com/(user)/(repo)/ - **branch** name is given as per the corresponding VCS (e.g., Git) as a query parameter to the requested URL. - **timestamp** is given in a format as liberal as possible, to uphold the principle of least surprise. At the very minimum it is possible to enter timestamps as: - Unix epoch timestamp (see for instance the output of `date +%s`) - ISO 8601 timestamps (see for instance the output of `date -I`, `date -Is`) - YYYY[MM[DD[HH[MM[SS]]]]] ad-hoc format - YYYY[-MM[-DD[ HH:[MM:[SS:]]]]] ad-hoc format swh-web Browse Urls ------------------- .. include:: uri-scheme-browse-content.rst .. include:: uri-scheme-browse-directory.rst .. include:: uri-scheme-browse-origin.rst .. include:: uri-scheme-browse-person.rst .. include:: uri-scheme-browse-release.rst .. include:: uri-scheme-browse-revision.rst .. include:: uri-scheme-browse-snapshot.rst diff --git a/swh/web/admin/origin_save.py b/swh/web/admin/origin_save.py index a9a50940..965a4600 100644 --- a/swh/web/admin/origin_save.py +++ b/swh/web/admin/origin_save.py @@ -1,206 +1,206 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.conf import settings from django.contrib.admin.views.decorators import staff_member_required from django.core.exceptions import ObjectDoesNotExist from django.core.paginator import Paginator from django.http import HttpResponse from django.shortcuts import render from django.views.decorators.http import require_POST from swh.web.admin.adminurls import admin_route from swh.web.common.models import ( SaveAuthorizedOrigin, SaveUnauthorizedOrigin, SaveOriginRequest ) from swh.web.common.origin_save import ( create_save_origin_request, get_save_origin_task_info, SAVE_REQUEST_PENDING, SAVE_REQUEST_REJECTED ) @admin_route(r'origin/save/', view_name='admin-origin-save') @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save(request): return render(request, 'admin/origin-save.html') def _datatables_origin_urls_response(request, urls_query_set): search_value = request.GET['search[value]'] if search_value: urls_query_set = urls_query_set.filter(url__icontains=search_value) column_order = request.GET['order[0][column]'] field_order = request.GET['columns[%s][name]' % column_order] order_dir = request.GET['order[0][dir]'] if order_dir == 'desc': field_order = '-' + field_order urls_query_set = urls_query_set.order_by(field_order) table_data = {} table_data['draw'] = int(request.GET['draw']) table_data['recordsTotal'] = urls_query_set.count() table_data['recordsFiltered'] = urls_query_set.count() length = int(request.GET['length']) page = int(request.GET['start']) / length + 1 paginator = Paginator(urls_query_set, length) urls_query_set = paginator.page(page).object_list table_data['data'] = [{'url': u.url} for u in urls_query_set] table_data_json = json.dumps(table_data, separators=(',', ': ')) return HttpResponse(table_data_json, content_type='application/json') @admin_route(r'origin/save/authorized_urls/list/', view_name='admin-origin-save-authorized-urls-list') @staff_member_required def _admin_origin_save_authorized_urls_list(request): authorized_urls = SaveAuthorizedOrigin.objects.all() return _datatables_origin_urls_response(request, authorized_urls) @admin_route(r'origin/save/authorized_urls/add/(?P.+)/', view_name='admin-origin-save-add-authorized-url') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_add_authorized_url(request, origin_url): try: SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: # add the new authorized url SaveAuthorizedOrigin.objects.create(url=origin_url) # check if pending save requests with that url prefix exist pending_save_requests = \ SaveOriginRequest.objects.filter(origin_url__startswith=origin_url, status=SAVE_REQUEST_PENDING) # create origin save tasks for previously pending requests for psr in pending_save_requests: - create_save_origin_request(psr.origin_type, psr.origin_url) + create_save_origin_request(psr.visit_type, psr.origin_url) status_code = 200 else: status_code = 400 return HttpResponse(status=status_code) @admin_route(r'origin/save/authorized_urls/remove/(?P.+)/', view_name='admin-origin-save-remove-authorized-url') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_remove_authorized_url(request, origin_url): try: entry = SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) @admin_route(r'origin/save/unauthorized_urls/list/', view_name='admin-origin-save-unauthorized-urls-list') @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_unauthorized_urls_list(request): unauthorized_urls = SaveUnauthorizedOrigin.objects.all() return _datatables_origin_urls_response(request, unauthorized_urls) @admin_route(r'origin/save/unauthorized_urls/add/(?P.+)/', view_name='admin-origin-save-add-unauthorized-url') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_add_unauthorized_url(request, origin_url): try: SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveUnauthorizedOrigin.objects.create(url=origin_url) # check if pending save requests with that url prefix exist pending_save_requests = \ SaveOriginRequest.objects.filter(origin_url__startswith=origin_url, status=SAVE_REQUEST_PENDING) # mark pending requests as rejected for psr in pending_save_requests: psr.status = SAVE_REQUEST_REJECTED psr.save() status_code = 200 else: status_code = 400 return HttpResponse(status=status_code) @admin_route(r'origin/save/unauthorized_urls/remove/(?P.+)/', view_name='admin-origin-save-remove-unauthorized-url') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_remove_unauthorized_url(request, origin_url): try: entry = SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) -@admin_route(r'origin/save/request/accept/(?P.+)/url/(?P.+)/', # noqa +@admin_route(r'origin/save/request/accept/(?P.+)/url/(?P.+)/', # noqa view_name='admin-origin-save-request-accept') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) -def _admin_origin_save_request_accept(request, origin_type, origin_url): +def _admin_origin_save_request_accept(request, visit_type, origin_url): try: SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveAuthorizedOrigin.objects.create(url=origin_url) - create_save_origin_request(origin_type, origin_url) + create_save_origin_request(visit_type, origin_url) return HttpResponse(status=200) -@admin_route(r'origin/save/request/reject/(?P.+)/url/(?P.+)/', # noqa +@admin_route(r'origin/save/request/reject/(?P.+)/url/(?P.+)/', # noqa view_name='admin-origin-save-request-reject') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) -def _admin_origin_save_request_reject(request, origin_type, origin_url): +def _admin_origin_save_request_reject(request, visit_type, origin_url): try: SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveUnauthorizedOrigin.objects.create(url=origin_url) - sor = SaveOriginRequest.objects.get(origin_type=origin_type, + sor = SaveOriginRequest.objects.get(visit_type=visit_type, origin_url=origin_url, status=SAVE_REQUEST_PENDING) sor.status = SAVE_REQUEST_REJECTED sor.save() return HttpResponse(status=200) @admin_route(r'origin/save/request/remove/(?P.+)/', view_name='admin-origin-save-request-remove') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_request_remove(request, sor_id): try: entry = SaveOriginRequest.objects.get(id=sor_id) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) @admin_route(r'origin/save/task/info/(?P.+)/', view_name='admin-origin-save-task-info') @staff_member_required(login_url=settings.LOGIN_URL) def _save_origin_task_info(request, save_request_id): request_info = get_save_origin_task_info(save_request_id) for date_field in ('scheduled', 'started', 'ended'): if date_field in request_info and request_info[date_field] is not None: request_info[date_field] = request_info[date_field].isoformat() return HttpResponse(json.dumps(request_info), content_type='application/json') diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py index 5720d1d0..94ef0958 100644 --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -1,543 +1,504 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from distutils.util import strtobool from functools import partial from swh.web.common import service from swh.web.common.exc import BadInputExc from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import reverse from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup DOC_RETURN_ORIGIN = ''' :>json string origin_visits_url: link to in order to get information about the visits for that origin :>json string url: the origin canonical url :>json string type: the type of software origin (deprecated value; types are now associated to visits instead of origins) :>json number id: the origin unique identifier (deprecated value; you should only refer to origins based on their URL) ''' DOC_RETURN_ORIGIN_ARRAY = \ DOC_RETURN_ORIGIN.replace(':>json', ':>jsonarr') DOC_RETURN_ORIGIN_VISIT = ''' :>json string date: ISO representation of the visit date (in UTC) :>json str origin: the origin canonical url :>json string origin_url: link to get information about the origin :>jsonarr string snapshot: the snapshot identifier of the visit (may be null if status is not **full**). :>jsonarr string snapshot_url: link to :http:get:`/api/1/snapshot/(snapshot_id)/` in order to get information about the snapshot of the visit (may be null if status is not **full**). :>json string status: status of the visit (either **full**, **partial** or **ongoing**) :>json number visit: the unique identifier of the visit ''' DOC_RETURN_ORIGIN_VISIT_ARRAY = \ DOC_RETURN_ORIGIN_VISIT.replace(':>json', ':>jsonarr') DOC_RETURN_ORIGIN_VISIT_ARRAY += ''' :>jsonarr number id: the unique identifier of the origin :>jsonarr string origin_visit_url: link to :http:get:`/api/1/origin/(origin_url)/visit/(visit_id)/` in order to get information about the visit ''' def _enrich_origin(origin): if 'url' in origin: o = origin.copy() o['origin_visits_url'] = reverse( 'api-1-origin-visits', url_args={'origin_url': origin['url']}) return o return origin def _enrich_origin_visit(origin_visit, *, with_origin_link, with_origin_visit_link): ov = origin_visit.copy() if with_origin_link: ov['origin_url'] = reverse('api-1-origin', url_args={'origin_url': ov['origin']}) if with_origin_visit_link: ov['origin_visit_url'] = reverse('api-1-origin-visit', url_args={'origin_url': ov['origin'], 'visit_id': ov['visit']}) snapshot = ov['snapshot'] if snapshot: ov['snapshot_url'] = reverse('api-1-snapshot', url_args={'snapshot_id': snapshot}) else: ov['snapshot_url'] = None return ov @api_route(r'/origins/', 'api-1-origins') @api_doc('/origins/', noargs=True) @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) def api_origins(request): """ .. http:get:: /api/1/origins/ Get list of archived software origins. .. warning:: This endpoint used to provide an `origin_from` query parameter, and guarantee an order on results. This is no longer true, and only the Link header should be used for paginating through results. :query int origin_count: The maximum number of origins to return (default to 100, can not exceed 10000) {return_origin_array} {common_headers} {resheader_link} **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origins?origin_count=500` """ origin_from = int(request.query_params.get('origin_from', '1')) origin_count = int(request.query_params.get('origin_count', '100')) origin_count = min(origin_count, 10000) results = api_lookup( service.lookup_origins, origin_from, origin_count+1, enrich_fn=_enrich_origin) response = {'results': results, 'headers': {}} if len(results) > origin_count: origin_from = results.pop()['id'] response['headers']['link-next'] = reverse( 'api-1-origins', query_params={'origin_from': origin_from, 'origin_count': origin_count}) return response -@api_route(r'/origin/(?P[a-z]+)/url/(?P.+)/', - 'api-1-origin') @api_route(r'/origin/(?P.+)/get/', 'api-1-origin') @api_doc('/origin/') @format_docstring(return_origin=DOC_RETURN_ORIGIN) -def api_origin(request, origin_url, origin_type=None): +def api_origin(request, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/get/ Get information about a software origin. :param string origin_url: the origin url {return_origin} {common_headers} **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: - :swh_web_api:`origin/git/url/https://github.com/python/cpython/` + :swh_web_api:`origin/https://github.com/python/cpython/get/` - - .. http:get:: /api/1/origin/(origin_type)/url/(origin_url)/ - - Get information about a software origin. - - .. warning:: - - All endpoints using an ``origin_type`` are - deprecated and will be removed in the near future. Only those - using only an ``origin_url`` will remain available. - You should use :http:get:`/api/1/origin/(origin_url)/get/` instead. - - :param string origin_type: the origin type (possible values are - ``git``, ``svn``, ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or - ``deposit``) - :param string origin_url: the origin url - - {return_origin} - - {common_headers} - - **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, - :http:method:`options` - - :statuscode 200: no error - :statuscode 404: requested origin can not be found in the archive - - **Example:** - - .. parsed-literal:: - - :swh_web_api:`origin/git/url/https://github.com/python/cpython/` """ ori_dict = { - 'type': origin_type, 'url': origin_url } - ori_dict = {k: v for k, v in ori_dict.items() if ori_dict[k]} - error_msg = 'Origin %s not found.' % \ - (ori_dict.get('id') or ori_dict['url']) + + error_msg = 'Origin with url %s not found.' % ori_dict['url'] return api_lookup( service.lookup_origin, ori_dict, notfound_msg=error_msg, enrich_fn=_enrich_origin) @api_route(r'/origin/search/(?P.+)/', 'api-1-origin-search') @api_doc('/origin/search/') @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) def api_origin_search(request, url_pattern): """ .. http:get:: /api/1/origin/search/(url_pattern)/ Search for software origins whose urls contain a provided string pattern or match a provided regular expression. The search is performed in a case insensitive way. .. warning:: This endpoint used to provide an `offset` query parameter, and guarantee an order on results. This is no longer true, and only the Link header should be used for paginating through results. :param string url_pattern: a string pattern or a regular expression :query int limit: the maximum number of found origins to return :query boolean regexp: if true, consider provided pattern as a regular expression and search origins whose urls match it :query boolean with_visit: if true, only return origins with at least one visit by Software heritage {return_origin_array} {common_headers} {resheader_link} **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origin/search/python/?limit=2` """ result = {} offset = int(request.query_params.get('offset', '0')) limit = int(request.query_params.get('limit', '70')) regexp = request.query_params.get('regexp', 'false') with_visit = request.query_params.get('with_visit', 'false') results = api_lookup(service.search_origin, url_pattern, offset, limit, bool(strtobool(regexp)), bool(strtobool(with_visit)), enrich_fn=_enrich_origin) nb_results = len(results) if nb_results == limit: query_params = {} query_params['offset'] = offset + limit query_params['limit'] = limit query_params['regexp'] = regexp result['headers'] = { 'link-next': reverse('api-1-origin-search', url_args={'url_pattern': url_pattern}, query_params=query_params) } result.update({ 'results': results }) return result @api_route(r'/origin/metadata-search/', 'api-1-origin-metadata-search') @api_doc('/origin/metadata-search/', noargs=True, need_params=True) @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) def api_origin_metadata_search(request): """ .. http:get:: /api/1/origin/metadata-search/ Search for software origins whose metadata (expressed as a JSON-LD/CodeMeta dictionary) match the provided criteria. For now, only full-text search on this dictionary is supported. :query str fulltext: a string that will be matched against origin metadata; results are ranked and ordered starting with the best ones. :query int limit: the maximum number of found origins to return (bounded to 100) {return_origin_array} {common_headers} **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origin/metadata-search/?limit=2&fulltext=Jane%20Doe` """ fulltext = request.query_params.get('fulltext', None) limit = min(int(request.query_params.get('limit', '70')), 100) if not fulltext: content = '"fulltext" must be provided and non-empty.' raise BadInputExc(content) results = api_lookup(service.search_origin_metadata, fulltext, limit) return { 'results': results, } @api_route(r'/origin/(?P.*)/visits/', 'api-1-origin-visits') @api_doc('/origin/visits/') @format_docstring( return_origin_visit_array=DOC_RETURN_ORIGIN_VISIT_ARRAY) def api_origin_visits(request, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/visits/ Get information about all visits of a software origin. Visits are returned sorted in descending order according to their date. :param str origin_url: a software origin URL :query int per_page: specify the number of visits to list, for pagination purposes :query int last_visit: visit to start listing from, for pagination purposes {common_headers} {resheader_link} {return_origin_visit_array} **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/https://github.com/hylang/hy/visits/` """ result = {} origin_query = {'url': origin_url} notfound_msg = 'No origin {} found'.format(origin_url) url_args_next = {'origin_url': origin_url} per_page = int(request.query_params.get('per_page', '10')) last_visit = request.query_params.get('last_visit') if last_visit: last_visit = int(last_visit) def _lookup_origin_visits( origin_query, last_visit=last_visit, per_page=per_page): all_visits = get_origin_visits(origin_query) all_visits.reverse() visits = [] if not last_visit: visits = all_visits[:per_page] else: for i, v in enumerate(all_visits): if v['visit'] == last_visit: visits = all_visits[i+1:i+1+per_page] break for v in visits: yield v results = api_lookup(_lookup_origin_visits, origin_query, notfound_msg=notfound_msg, enrich_fn=partial(_enrich_origin_visit, with_origin_link=False, with_origin_visit_link=True)) if results: nb_results = len(results) if nb_results == per_page: new_last_visit = results[-1]['visit'] query_params = {} query_params['last_visit'] = new_last_visit if request.query_params.get('per_page'): query_params['per_page'] = per_page result['headers'] = { 'link-next': reverse('api-1-origin-visits', url_args=url_args_next, query_params=query_params) } result.update({ 'results': results }) return result @api_route(r'/origin/(?P.*)/visit/latest/', 'api-1-origin-visit-latest', throttle_scope='swh_api_origin_visit_latest') @api_doc('/origin/visit/latest/') @format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) def api_origin_visit_latest(request, origin_url=None): """ .. http:get:: /api/1/origin/(origin_url)/visit/latest/ Get information about the latest visit of a software origin. :param str origin_url: a software origin URL :query boolean require_snapshot: if true, only return a visit with a snapshot {common_headers} {return_origin_visit} **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin or visit can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/https://github.com/hylang/hy/visit/latest/` """ require_snapshot = request.query_params.get('require_snapshot', 'false') return api_lookup( service.lookup_origin_visit_latest, origin_url, bool(strtobool(require_snapshot)), notfound_msg=('No visit for origin {} found' .format(origin_url)), enrich_fn=partial(_enrich_origin_visit, with_origin_link=True, with_origin_visit_link=False)) @api_route(r'/origin/(?P.*)/visit/(?P[0-9]+)/', 'api-1-origin-visit') @api_doc('/origin/visit/') @format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) def api_origin_visit(request, visit_id, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/ Get information about a specific visit of a software origin. :param str origin_url: a software origin URL :param int visit_id: a visit identifier {common_headers} {return_origin_visit} **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin or visit can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/https://github.com/hylang/hy/visit/1/` """ return api_lookup( service.lookup_origin_visit, origin_url, int(visit_id), notfound_msg=('No visit {} for origin {} found' .format(visit_id, origin_url)), enrich_fn=partial(_enrich_origin_visit, with_origin_link=True, with_origin_visit_link=False)) -@api_route(r'/origin/(?P[a-z]+)/url/(?P.+)' +@api_route(r'/origin/(?P.+)' '/intrinsic-metadata', 'api-origin-intrinsic-metadata') @api_doc('/origin/intrinsic-metadata/') @format_docstring() -def api_origin_intrinsic_metadata(request, origin_type, origin_url): +def api_origin_intrinsic_metadata(request, origin_url): """ - .. http:get:: /api/1/origin/(origin_type)/url/(origin_url)/intrinsic-metadata + .. http:get:: /api/1/origin/(origin_url)/intrinsic-metadata Get intrinsic metadata of a software origin (as a JSON-LD/CodeMeta dictionary). - :param string origin_type: the origin type (possible values are ``git``, ``svn``, - ``hg``, ``deb``, ``pypi``, ``npm``, ``ftp`` or ``deposit``) :param string origin_url: the origin url :>json string ???: intrinsic metadata field of the origin {common_headers} **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: - :swh_web_api:`origin/git/url/https://github.com/python/cpython/intrinsic-metadata` + :swh_web_api:`origin/https://github.com/python/cpython/intrinsic-metadata` """ # noqa ori_dict = { - 'type': origin_type, 'url': origin_url } - error_msg = 'Origin with URL %s not found' % ori_dict['url'] + error_msg = 'Origin with url %s not found' % ori_dict['url'] return api_lookup( service.lookup_origin_intrinsic_metadata, ori_dict, notfound_msg=error_msg, enrich_fn=_enrich_origin) diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py index 6ac43fe1..be6240ed 100644 --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -1,87 +1,88 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.views.decorators.cache import never_cache from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.common.origin_save import ( create_save_origin_request, get_save_origin_requests ) -@api_route(r'/origin/save/(?P.+)/url/(?P.+)/', +@api_route(r'/origin/save/(?P.+)/url/(?P.+)/', 'api-1-save-origin', methods=['GET', 'POST'], throttle_scope='swh_save_origin') @never_cache @api_doc('/origin/save/') @format_docstring() -def api_save_origin(request, origin_type, origin_url): +def api_save_origin(request, visit_type, origin_url): """ - .. http:get:: /api/1/origin/save/(origin_type)/url/(origin_url)/ - .. http:post:: /api/1/origin/save/(origin_type)/url/(origin_url)/ + .. http:get:: /api/1/origin/save/(visit_type)/url/(origin_url)/ + .. http:post:: /api/1/origin/save/(visit_type)/url/(origin_url)/ Request the saving of a software origin into the archive or check the status of previously created save requests. That endpoint enables to create a saving task for a software origin through a POST request. Depending of the provided origin url, the save request can either be: * immediately **accepted**, for well known code hosting providers like for instance GitHub or GitLab * **rejected**, in case the url is blacklisted by Software Heritage * **put in pending state** until a manual check is done in order to determine if it can be loaded or not Once a saving request has been accepted, its associated saving task status can then be checked through a GET request on the same url. Returned status can either be: * **not created**: no saving task has been created * **not yet scheduled**: saving task has been created but its execution has not yet been scheduled * **scheduled**: the task execution has been scheduled * **succeed**: the saving task has been successfully executed * **failed**: the saving task has been executed but it failed When issuing a POST request an object will be returned while a GET request will return an array of objects (as multiple save requests might have been submitted for the same origin). - :param string origin_type: the type of origin to save + :param string visit_type: the type of visit to perform (currently the supported types are ``git``, ``hg`` and ``svn``) :param string origin_url: the url of the origin to save {common_headers} :>json string origin_url: the url of the origin to save - :>json string origin_type: the type of the origin to save + :>json string visit_type: the type of visit to perform :>json string save_request_date: the date (in iso format) the save request was issued :>json string save_request_status: the status of the save request, either **accepted**, **rejected** or **pending** :>json string save_task_status: the status of the origin saving task, either **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** **Allowed HTTP Methods:** :http:method:`get`, :http:method:`post`, :http:method:`head`, :http:method:`options` :statuscode 200: no error - :statuscode 400: an invalid origin type or url has been provided + :statuscode 400: an invalid visit type or origin url has been provided :statuscode 403: the provided origin url is blacklisted :statuscode 404: no save requests have been found for a given origin """ if request.method == 'POST': - sor = create_save_origin_request(origin_type, origin_url) + sor = create_save_origin_request(visit_type, origin_url) del sor['id'] else: - sor = get_save_origin_requests(origin_type, origin_url) - for s in sor: del s['id'] # noqa + sor = get_save_origin_requests(visit_type, origin_url) + for s in sor: + del s['id'] return sor diff --git a/swh/web/assets/src/bundles/admin/origin-save.js b/swh/web/assets/src/bundles/admin/origin-save.js index 14b3d265..8afaa478 100644 --- a/swh/web/assets/src/bundles/admin/origin-save.js +++ b/swh/web/assets/src/bundles/admin/origin-save.js @@ -1,450 +1,450 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {handleFetchError, csrfPost, htmlAlert} from 'utils/functions'; import {swhSpinnerSrc} from 'utils/constants'; let authorizedOriginTable; let unauthorizedOriginTable; let pendingSaveRequestsTable; let acceptedSaveRequestsTable; let rejectedSaveRequestsTable; function enableRowSelection(tableSel) { $(`${tableSel} tbody`).on('click', 'tr', function() { if ($(this).hasClass('selected')) { $(this).removeClass('selected'); $(tableSel).closest('.tab-pane').find('.swh-action-need-selection').prop('disabled', true); } else { $(`${tableSel} tr.selected`).removeClass('selected'); $(this).addClass('selected'); $(tableSel).closest('.tab-pane').find('.swh-action-need-selection').prop('disabled', false); } }); } export function initOriginSaveAdmin() { $(document).ready(() => { $.fn.dataTable.ext.errMode = 'throw'; authorizedOriginTable = $('#swh-authorized-origin-urls').DataTable({ serverSide: true, ajax: Urls.admin_origin_save_authorized_urls_list(), columns: [{data: 'url', name: 'url'}], scrollY: '50vh', scrollCollapse: true, info: false }); enableRowSelection('#swh-authorized-origin-urls'); swh.webapp.addJumpToPagePopoverToDataTable(authorizedOriginTable); unauthorizedOriginTable = $('#swh-unauthorized-origin-urls').DataTable({ serverSide: true, ajax: Urls.admin_origin_save_unauthorized_urls_list(), columns: [{data: 'url', name: 'url'}], scrollY: '50vh', scrollCollapse: true, info: false }); enableRowSelection('#swh-unauthorized-origin-urls'); swh.webapp.addJumpToPagePopoverToDataTable(unauthorizedOriginTable); let columnsData = [ { data: 'id', name: 'id', visible: false, searchable: false }, { data: 'save_request_date', name: 'request_date', render: (data, type, row) => { if (type === 'display') { let date = new Date(data); return date.toLocaleString(); } return data; } }, { - data: 'origin_type', - name: 'origin_type' + data: 'visit_type', + name: 'visit_type' }, { data: 'origin_url', name: 'origin_url', render: (data, type, row) => { if (type === 'display') { const sanitizedURL = $.fn.dataTable.render.text().display(data); return `${sanitizedURL}`; } return data; } } ]; pendingSaveRequestsTable = $('#swh-origin-save-pending-requests').DataTable({ serverSide: true, processing: true, language: { processing: `` }, ajax: Urls.origin_save_requests_list('pending'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-pending-requests'); swh.webapp.addJumpToPagePopoverToDataTable(pendingSaveRequestsTable); rejectedSaveRequestsTable = $('#swh-origin-save-rejected-requests').DataTable({ serverSide: true, processing: true, language: { processing: `` }, ajax: Urls.origin_save_requests_list('rejected'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-rejected-requests'); swh.webapp.addJumpToPagePopoverToDataTable(rejectedSaveRequestsTable); columnsData.push({ data: 'save_task_status', name: 'save_task_status', render: (data, type, row) => { if (data === 'succeed' && row.visit_date) { let browseOriginUrl = Urls.browse_origin(row.origin_url); browseOriginUrl += `visit/${row.visit_date}/`; return `${data}`; } return data; } }); columnsData.push({ name: 'info', render: (data, type, row) => { if (row.save_task_status === 'succeed' || row.save_task_status === 'failed') { return '`; } else { return ''; } } }); acceptedSaveRequestsTable = $('#swh-origin-save-accepted-requests').DataTable({ serverSide: true, processing: true, language: { processing: `` }, ajax: Urls.origin_save_requests_list('accepted'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-accepted-requests'); swh.webapp.addJumpToPagePopoverToDataTable(acceptedSaveRequestsTable); $('#swh-origin-save-requests-nav-item').on('shown.bs.tab', () => { pendingSaveRequestsTable.draw(); }); $('#swh-origin-save-url-filters-nav-item').on('shown.bs.tab', () => { authorizedOriginTable.draw(); }); $('#swh-authorized-origins-tab').on('shown.bs.tab', () => { authorizedOriginTable.draw(); }); $('#swh-unauthorized-origins-tab').on('shown.bs.tab', () => { unauthorizedOriginTable.draw(); }); $('#swh-save-requests-pending-tab').on('shown.bs.tab', () => { pendingSaveRequestsTable.draw(); }); $('#swh-save-requests-accepted-tab').on('shown.bs.tab', () => { acceptedSaveRequestsTable.draw(); }); $('#swh-save-requests-rejected-tab').on('shown.bs.tab', () => { rejectedSaveRequestsTable.draw(); }); $('#swh-save-requests-pending-tab').click(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); $('#swh-save-requests-accepted-tab').click(() => { acceptedSaveRequestsTable.ajax.reload(null, false); }); $('#swh-save-requests-rejected-tab').click(() => { rejectedSaveRequestsTable.ajax.reload(null, false); }); $('body').on('click', e => { if ($(e.target).parents('.popover').length > 0) { event.stopPropagation(); } else if ($(e.target).parents('.swh-save-request-info').length === 0) { $('.swh-save-request-info').popover('dispose'); } }); }); } export function addAuthorizedOriginUrl() { let originUrl = $('#swh-authorized-url-prefix').val(); let addOriginUrl = Urls.admin_origin_save_add_authorized_url(originUrl); csrfPost(addOriginUrl) .then(handleFetchError) .then(() => { authorizedOriginTable.row.add({'url': originUrl}).draw(); $('.swh-add-authorized-origin-status').html( htmlAlert('success', 'The origin url prefix has been successfully added in the authorized list.', true) ); }) .catch(response => { $('.swh-add-authorized-origin-status').html( htmlAlert('warning', 'The provided origin url prefix is already registered in the authorized list.', true) ); }); } export function removeAuthorizedOriginUrl() { let originUrl = $('#swh-authorized-origin-urls tr.selected').text(); if (originUrl) { let removeOriginUrl = Urls.admin_origin_save_remove_authorized_url(originUrl); csrfPost(removeOriginUrl) .then(handleFetchError) .then(() => { authorizedOriginTable.row('.selected').remove().draw(); }) .catch(() => {}); } } export function addUnauthorizedOriginUrl() { let originUrl = $('#swh-unauthorized-url-prefix').val(); let addOriginUrl = Urls.admin_origin_save_add_unauthorized_url(originUrl); csrfPost(addOriginUrl) .then(handleFetchError) .then(() => { unauthorizedOriginTable.row.add({'url': originUrl}).draw(); $('.swh-add-unauthorized-origin-status').html( htmlAlert('success', 'The origin url prefix has been successfully added in the unauthorized list.', true) ); }) .catch(() => { $('.swh-add-unauthorized-origin-status').html( htmlAlert('warning', 'The provided origin url prefix is already registered in the unauthorized list.', true) ); }); } export function removeUnauthorizedOriginUrl() { let originUrl = $('#swh-unauthorized-origin-urls tr.selected').text(); if (originUrl) { let removeOriginUrl = Urls.admin_origin_save_remove_unauthorized_url(originUrl); csrfPost(removeOriginUrl) .then(handleFetchError) .then(() => { unauthorizedOriginTable.row('.selected').remove().draw(); }) .catch(() => {}); } } export function acceptOriginSaveRequest() { let selectedRow = pendingSaveRequestsTable.row('.selected'); if (selectedRow.length) { let acceptOriginSaveRequestCallback = () => { let rowData = selectedRow.data(); - let acceptSaveRequestUrl = Urls.admin_origin_save_request_accept(rowData['origin_type'], rowData['origin_url']); + let acceptSaveRequestUrl = Urls.admin_origin_save_request_accept(rowData['visit_type'], rowData['origin_url']); csrfPost(acceptSaveRequestUrl) .then(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Accept origin save request ?', 'Are you sure to accept this origin save request ?', acceptOriginSaveRequestCallback); } } export function rejectOriginSaveRequest() { let selectedRow = pendingSaveRequestsTable.row('.selected'); if (selectedRow.length) { let rejectOriginSaveRequestCallback = () => { let rowData = selectedRow.data(); - let rejectSaveRequestUrl = Urls.admin_origin_save_request_reject(rowData['origin_type'], rowData['origin_url']); + let rejectSaveRequestUrl = Urls.admin_origin_save_request_reject(rowData['visit_type'], rowData['origin_url']); csrfPost(rejectSaveRequestUrl) .then(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Reject origin save request ?', 'Are you sure to reject this origin save request ?', rejectOriginSaveRequestCallback); } } function removeOriginSaveRequest(requestTable) { let selectedRow = requestTable.row('.selected'); if (selectedRow.length) { let requestId = selectedRow.data()['id']; let removeOriginSaveRequestCallback = () => { let removeSaveRequestUrl = Urls.admin_origin_save_request_remove(requestId); csrfPost(removeSaveRequestUrl) .then(() => { requestTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Remove origin save request ?', 'Are you sure to remove this origin save request ?', removeOriginSaveRequestCallback); } } export function removePendingOriginSaveRequest() { removeOriginSaveRequest(pendingSaveRequestsTable); } export function removeAcceptedOriginSaveRequest() { removeOriginSaveRequest(acceptedSaveRequestsTable); } export function removeRejectedOriginSaveRequest() { removeOriginSaveRequest(rejectedSaveRequestsTable); } export function displaySaveRequestInfo(event, saveRequestId) { event.stopPropagation(); const saveRequestTaskInfoUrl = Urls.admin_origin_save_task_info(saveRequestId); $('.swh-save-request-info').popover('dispose'); $(event.target).popover({ 'title': 'Save request task information', 'content': `

Fetching task information ...

`, 'html': true, 'placement': 'left', 'sanitizeFn': swh.webapp.filterXSS }); $(event.target).popover('show'); fetch(saveRequestTaskInfoUrl) .then(response => response.json()) .then(saveRequestTaskInfo => { let content; if ($.isEmptyObject(saveRequestTaskInfo)) { content = 'Not available'; } else { let saveRequestInfo = []; saveRequestInfo.push({ key: 'Task type', value: saveRequestTaskInfo.type }); if (saveRequestTaskInfo.hasOwnProperty('task_name')) { saveRequestInfo.push({ key: 'Task name', value: saveRequestTaskInfo.name }); } saveRequestInfo.push({ key: 'Task arguments', value: JSON.stringify(saveRequestTaskInfo.arguments, null, 2) }); saveRequestInfo.push({ key: 'Task id', value: saveRequestTaskInfo.id }); saveRequestInfo.push({ key: 'Task backend id', value: saveRequestTaskInfo.backend_id }); saveRequestInfo.push({ key: 'Task scheduling date', value: new Date(saveRequestTaskInfo.scheduled).toLocaleString() }); saveRequestInfo.push({ key: 'Task termination date', value: new Date(saveRequestTaskInfo.ended).toLocaleString() }); if (saveRequestTaskInfo.hasOwnProperty('duration')) { saveRequestInfo.push({ key: 'Task duration', value: saveRequestTaskInfo.duration + ' s' }); } if (saveRequestTaskInfo.hasOwnProperty('worker')) { saveRequestInfo.push({ key: 'Task executor', value: saveRequestTaskInfo.worker }); } if (saveRequestTaskInfo.hasOwnProperty('message')) { saveRequestInfo.push({ key: 'Task log', value: saveRequestTaskInfo.message }); } content = ''; for (let info of saveRequestInfo) { content += ``; } content += '
${info.key} 
${info.value}
'; } $('.swh-popover').html(content); $(event.target).popover('update'); }); } diff --git a/swh/web/assets/src/bundles/browse/origin-search.js b/swh/web/assets/src/bundles/browse/origin-search.js index 50f99bb7..c1d2efe3 100644 --- a/swh/web/assets/src/bundles/browse/origin-search.js +++ b/swh/web/assets/src/bundles/browse/origin-search.js @@ -1,232 +1,233 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {heapsPermute} from 'utils/heaps-permute'; import {handleFetchError} from 'utils/functions'; let originPatterns; let perPage = 100; let limit = perPage * 2; let offset = 0; let currentData = null; let inSearch = false; function fixTableRowsStyle() { setTimeout(() => { $('#origin-search-results tbody tr').removeAttr('style'); }); } function clearOriginSearchResultsTable() { $('#origin-search-results tbody tr').remove(); } function populateOriginSearchResultsTable(origins, offset) { let localOffset = offset % limit; if (origins.length > 0) { $('#swh-origin-search-results').show(); $('#swh-no-result').hide(); clearOriginSearchResultsTable(); let table = $('#origin-search-results tbody'); for (let i = localOffset; i < localOffset + perPage && i < origins.length; ++i) { let origin = origins[i]; let browseUrl = Urls.browse_origin(origin.url); let tableRow = ``; - tableRow += `${origin.type}`; tableRow += `${encodeURI(origin.url)}`; + tableRow += ``; tableRow += ``; tableRow += ''; table.append(tableRow); // get async latest visit snapshot and update visit status icon let latestSnapshotUrl = Urls.api_1_origin_visit_latest(origin.url); latestSnapshotUrl += '?require_snapshot=true'; fetch(latestSnapshotUrl) .then(response => response.json()) .then(data => { + $(`#visit-type-origin-${i}`).text(data.type); $(`#visit-status-origin-${i}`).children().remove(); if (data) { $(`#visit-status-origin-${i}`).append(''); } else { $(`#visit-status-origin-${i}`).append(''); if ($('#swh-filter-empty-visits').prop('checked')) { $(`#origin-${i}`).remove(); } } }); } fixTableRowsStyle(); } else { $('#swh-origin-search-results').hide(); $('#swh-no-result').text('No origins matching the search criteria were found.'); $('#swh-no-result').show(); } if (origins.length - localOffset < perPage || (origins.length < limit && (localOffset + perPage) === origins.length)) { $('#origins-next-results-button').addClass('disabled'); } else { $('#origins-next-results-button').removeClass('disabled'); } if (offset > 0) { $('#origins-prev-results-button').removeClass('disabled'); } else { $('#origins-prev-results-button').addClass('disabled'); } inSearch = false; setTimeout(() => { window.scrollTo(0, 0); }); } function escapeStringRegexp(str) { let matchOperatorsRe = /[|\\{}()[\]^$+*?.]/g; return str.replace(matchOperatorsRe, '\\\\\\$&'); } function searchOrigins(patterns, limit, searchOffset, offset) { let baseSearchUrl; let searchMetadata = $('#swh-search-origin-metadata').prop('checked'); if (searchMetadata) { baseSearchUrl = Urls.api_1_origin_metadata_search() + `?fulltext=${patterns}`; } else { originPatterns = patterns; let patternsArray = patterns.trim().replace(/\s+/g, ' ').split(' '); for (let i = 0; i < patternsArray.length; ++i) { patternsArray[i] = escapeStringRegexp(patternsArray[i]); } // url length must be less than 4096 for modern browsers // assuming average word length, 6 is max patternArray.length if (patternsArray.length < 7) { let patternsPermut = []; heapsPermute(patternsArray, p => patternsPermut.push(p.join('.*'))); let regex = patternsPermut.join('|'); baseSearchUrl = Urls.browse_origin_search(regex) + `?regexp=true`; } else { baseSearchUrl = Urls.browse_origin_search(patternsArray.join('.*')) + `?regexp=true`; } } let withVisit = $('#swh-search-origins-with-visit').prop('checked'); let searchUrl = baseSearchUrl + `&limit=${limit}&offset=${searchOffset}&with_visit=${withVisit}`; clearOriginSearchResultsTable(); $('.swh-loading').addClass('show'); fetch(searchUrl) .then(handleFetchError) .then(response => response.json()) .then(data => { currentData = data; $('.swh-loading').removeClass('show'); populateOriginSearchResultsTable(data, offset); }) .catch(response => { $('.swh-loading').removeClass('show'); inSearch = false; $('#swh-origin-search-results').hide(); $('#swh-no-result').text(`Error ${response.status}: ${response.statusText}`); $('#swh-no-result').show(); }); } function doSearch() { $('#swh-no-result').hide(); let patterns = $('#origins-url-patterns').val(); offset = 0; inSearch = true; // first try to resolve a swh persistent identifier let resolvePidUrl = Urls.api_1_resolve_swh_pid(patterns); fetch(resolvePidUrl) .then(handleFetchError) .then(response => response.json()) .then(data => { // pid has been successfully resolved, // so redirect to browse page window.location = data.browse_url; }) .catch(response => { // pid resolving failed if (patterns.startsWith('swh:')) { // display a useful error message if the input // looks like a swh pid response.json().then(data => { $('#swh-origin-search-results').hide(); $('.swh-search-pagination').hide(); $('#swh-no-result').text(data.reason); $('#swh-no-result').show(); }); } else { // otherwise, proceed with origins search $('#swh-origin-search-results').show(); $('.swh-search-pagination').show(); searchOrigins(patterns, limit, offset, offset); } }); } export function initOriginSearch() { $(document).ready(() => { $('#swh-search-origins').submit(event => { event.preventDefault(); let patterns = $('#origins-url-patterns').val().trim(); let withVisit = $('#swh-search-origins-with-visit').prop('checked'); let withContent = $('#swh-filter-empty-visits').prop('checked'); let searchMetadata = $('#swh-search-origin-metadata').prop('checked'); let queryParameters = '?q=' + encodeURIComponent(patterns); if (withVisit) { queryParameters += '&with_visit'; } if (withContent) { queryParameters += '&with_content'; } if (searchMetadata) { queryParameters += '&search_metadata'; } // Update the url, triggering page reload and effective search window.location.search = queryParameters; }); $('#origins-next-results-button').click(event => { if ($('#origins-next-results-button').hasClass('disabled') || inSearch) { return; } inSearch = true; offset += perPage; if (!currentData || (offset >= limit && offset % limit === 0)) { searchOrigins(originPatterns, limit, offset, offset); } else { populateOriginSearchResultsTable(currentData, offset); } event.preventDefault(); }); $('#origins-prev-results-button').click(event => { if ($('#origins-prev-results-button').hasClass('disabled') || inSearch) { return; } inSearch = true; offset -= perPage; if (!currentData || (offset > 0 && (offset + perPage) % limit === 0)) { searchOrigins(originPatterns, limit, (offset + perPage) - limit, offset); } else { populateOriginSearchResultsTable(currentData, offset); } event.preventDefault(); }); let urlParams = new URLSearchParams(window.location.search); let query = urlParams.get('q'); let withVisit = urlParams.has('with_visit'); let withContent = urlParams.has('with_content'); let searchMetadata = urlParams.has('search_metadata'); if (query) { $('#origins-url-patterns').val(query); $('#swh-search-origins-with-visit').prop('checked', withVisit); $('#swh-filter-empty-visits').prop('checked', withContent); $('#swh-search-origin-metadata').prop('checked', searchMetadata); doSearch(); } }); } diff --git a/swh/web/assets/src/bundles/save/index.js b/swh/web/assets/src/bundles/save/index.js index 641ebd15..b6afa48c 100644 --- a/swh/web/assets/src/bundles/save/index.js +++ b/swh/web/assets/src/bundles/save/index.js @@ -1,297 +1,297 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {handleFetchError, csrfPost, isGitRepoUrl, htmlAlert, removeUrlFragment} from 'utils/functions'; import {swhSpinnerSrc} from 'utils/constants'; import {validate} from 'validate.js'; let saveRequestsTable; function originSaveRequest(originType, originUrl, acceptedCallback, pendingCallback, errorCallback) { let addSaveOriginRequestUrl = Urls.origin_save_request(originType, originUrl); let headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' }; $('.swh-processing-save-request').css('display', 'block'); csrfPost(addSaveOriginRequestUrl, headers) .then(handleFetchError) .then(response => response.json()) .then(data => { $('.swh-processing-save-request').css('display', 'none'); if (data.save_request_status === 'accepted') { acceptedCallback(); } else { pendingCallback(); } }) .catch(response => { $('.swh-processing-save-request').css('display', 'none'); errorCallback(response.status); }); } export function initOriginSave() { $(document).ready(() => { $.fn.dataTable.ext.errMode = 'none'; fetch(Urls.origin_save_types_list()) .then(response => response.json()) .then(data => { for (let originType of data) { - $('#swh-input-origin-type').append(``); + $('#swh-input-visit-type').append(``); } }); saveRequestsTable = $('#swh-origin-save-requests') .on('error.dt', (e, settings, techNote, message) => { $('#swh-origin-save-request-list-error').text('An error occurred while retrieving the save requests list'); console.log(message); }) .DataTable({ serverSide: true, processing: true, language: { processing: `` }, ajax: Urls.origin_save_requests_list('all'), searchDelay: 1000, columns: [ { data: 'save_request_date', name: 'request_date', render: (data, type, row) => { if (type === 'display') { let date = new Date(data); return date.toLocaleString(); } return data; } }, { - data: 'origin_type', - name: 'origin_type' + data: 'visit_type', + name: 'visit_type' }, { data: 'origin_url', name: 'origin_url', render: (data, type, row) => { if (type === 'display') { const sanitizedURL = $.fn.dataTable.render.text().display(data); return `${sanitizedURL}`; } return data; } }, { data: 'save_request_status', name: 'status' }, { data: 'save_task_status', name: 'loading_task_status', render: (data, type, row) => { if (data === 'succeed' && row.visit_date) { let browseOriginUrl = Urls.browse_origin(row.origin_url); browseOriginUrl += `visit/${row.visit_date}/`; return `${data}`; } return data; } } ], scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); swh.webapp.addJumpToPagePopoverToDataTable(saveRequestsTable); $('#swh-origin-save-requests-list-tab').on('shown.bs.tab', () => { saveRequestsTable.draw(); window.location.hash = '#requests'; }); $('#swh-origin-save-request-create-tab').on('shown.bs.tab', () => { removeUrlFragment(); }); let saveRequestAcceptedAlert = htmlAlert( 'success', 'The "save code now" request has been accepted and will be processed as soon as possible.' ); let saveRequestPendingAlert = htmlAlert( 'warning', 'The "save code now" request has been put in pending state and may be accepted for processing after manual review.' ); let saveRequestRejectedAlert = htmlAlert( 'danger', 'The "save code now" request has been rejected because the provided origin url is blacklisted.' ); let saveRequestRateLimitedAlert = htmlAlert( 'danger', 'The rate limit for "save code now" requests has been reached. Please try again later.' ); let saveRequestUnknownErrorAlert = htmlAlert( 'danger', 'An unexpected error happened when submitting the "save code now request' ); $('#swh-save-origin-form').submit(event => { event.preventDefault(); event.stopPropagation(); $('.alert').alert('close'); if (event.target.checkValidity()) { $(event.target).removeClass('was-validated'); - let originType = $('#swh-input-origin-type').val(); + let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); originSaveRequest(originType, originUrl, () => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert), () => $('#swh-origin-save-request-status').html(saveRequestPendingAlert), (statusCode) => { $('#swh-origin-save-request-status').css('color', 'red'); if (statusCode === 403) { $('#swh-origin-save-request-status').html(saveRequestRejectedAlert); } else if (statusCode === 429) { $('#swh-origin-save-request-status').html(saveRequestRateLimitedAlert); } else { $('#swh-origin-save-request-status').html(saveRequestUnknownErrorAlert); } }); } else { $(event.target).addClass('was-validated'); } }); $('#swh-show-origin-save-requests-list').on('click', (event) => { event.preventDefault(); $('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show'); }); $('#swh-input-origin-url').on('input', function(event) { let originUrl = $(this).val().trim(); $(this).val(originUrl); - $('#swh-input-origin-type option').each(function() { + $('#swh-input-visit-type option').each(function() { let val = $(this).val(); if (val && originUrl.includes(val)) { $(this).prop('selected', true); } }); }); if (window.location.hash === '#requests') { $('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show'); } }); } export function validateSaveOriginUrl(input) { let originUrl = input.value.trim(); let validUrl = validate({website: originUrl}, { website: { url: { schemes: ['http', 'https', 'svn', 'git'] } } }) === undefined; - let originType = $('#swh-input-origin-type').val(); + let originType = $('#swh-input-visit-type').val(); if (originType === 'git' && validUrl) { // additional checks for well known code hosting providers let githubIdx = originUrl.indexOf('://github.com'); let gitlabIdx = originUrl.indexOf('://gitlab.'); let gitSfIdx = originUrl.indexOf('://git.code.sf.net'); let bitbucketIdx = originUrl.indexOf('://bitbucket.org'); if (githubIdx !== -1 && githubIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'github.com'); } else if (gitlabIdx !== -1 && gitlabIdx <= 5) { let startIdx = gitlabIdx + 3; let idx = originUrl.indexOf('/', startIdx); if (idx !== -1) { let gitlabDomain = originUrl.substr(startIdx, idx - startIdx); validUrl = isGitRepoUrl(originUrl, gitlabDomain); } else { validUrl = false; } } else if (gitSfIdx !== -1 && gitSfIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'git.code.sf.net/p'); } else if (bitbucketIdx !== -1 && bitbucketIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'bitbucket.org'); } } if (validUrl) { input.setCustomValidity(''); } else { input.setCustomValidity('The origin url is not valid or does not reference a code repository'); } } export function initTakeNewSnapshot() { let newSnapshotRequestAcceptedAlert = htmlAlert( 'success', 'The "take new snapshot" request has been accepted and will be processed as soon as possible.' ); let newSnapshotRequestPendingAlert = htmlAlert( 'warning', 'The "take new snapshot" request has been put in pending state and may be accepted for processing after manual review.' ); let newSnapshotRequestRejectedAlert = htmlAlert( 'danger', 'The "take new snapshot" request has been rejected.' ); let newSnapshotRequestRateLimitAlert = htmlAlert( 'danger', 'The rate limit for "take new snapshot" requests has been reached. Please try again later.' ); let newSnapshotRequestUnknownErrorAlert = htmlAlert( 'danger', 'An unexpected error happened when submitting the "save code now request".' ); $(document).ready(() => { $('#swh-take-new-snapshot-form').submit(event => { event.preventDefault(); event.stopPropagation(); - let originType = $('#swh-input-origin-type').val(); + let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); originSaveRequest(originType, originUrl, () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), (statusCode) => { $('#swh-take-new-snapshot-request-status').css('color', 'red'); if (statusCode === 403) { $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRejectedAlert); } else if (statusCode === 429) { $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRateLimitAlert); } else { $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestUnknownErrorAlert); } }); }); }); } diff --git a/swh/web/browse/utils.py b/swh/web/browse/utils.py index dcbe2d1b..5e544546 100644 --- a/swh/web/browse/utils.py +++ b/swh/web/browse/utils.py @@ -1,1139 +1,1098 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import base64 import magic import pypandoc import stat import textwrap from collections import defaultdict from threading import Lock from django.core.cache import cache from django.utils.safestring import mark_safe from django.utils.html import escape from swh.model.identifiers import persistent_identifier from swh.web.common import highlightjs, service from swh.web.common.exc import NotFoundExc, http_status_code_message from swh.web.common.origin_visits import get_origin_visit from swh.web.common.utils import ( reverse, format_utc_iso_date, get_swh_persistent_id, swh_object_icons ) from swh.web.config import get_config def get_directory_entries(sha1_git): """Function that retrieves the content of a directory from the archive. The directories entries are first sorted in lexicographical order. Sub-directories and regular files are then extracted. Args: sha1_git: sha1_git identifier of the directory Returns: A tuple whose first member corresponds to the sub-directories list and second member the regular files list Raises: NotFoundExc if the directory is not found """ cache_entry_id = 'directory_entries_%s' % sha1_git cache_entry = cache.get(cache_entry_id) if cache_entry: return cache_entry entries = list(service.lookup_directory(sha1_git)) for e in entries: e['perms'] = stat.filemode(e['perms']) if e['type'] == 'rev': # modify dir entry name to explicitly show it points # to a revision e['name'] = '%s @ %s' % (e['name'], e['target'][:7]) dirs = [e for e in entries if e['type'] in ('dir', 'rev')] files = [e for e in entries if e['type'] == 'file'] dirs = sorted(dirs, key=lambda d: d['name']) files = sorted(files, key=lambda f: f['name']) cache.set(cache_entry_id, (dirs, files)) return dirs, files _lock = Lock() def get_mimetype_and_encoding_for_content(content): """Function that returns the mime type and the encoding associated to a content buffer using the magic module under the hood. Args: content (bytes): a content buffer Returns: A tuple (mimetype, encoding), for instance ('text/plain', 'us-ascii'), associated to the provided content. """ # https://pypi.org/project/python-magic/ # packaged as python3-magic in debian buster if hasattr(magic, 'from_buffer'): m = magic.Magic(mime=True, mime_encoding=True) mime_encoding = m.from_buffer(content) mime_type, encoding = mime_encoding.split(';') encoding = encoding.replace(' charset=', '') # https://pypi.org/project/file-magic/ # packaged as python3-magic in debian stretch else: # TODO: Remove that code when production environment is upgraded # to debian buster # calls to the file-magic API are not thread-safe so they must # be protected with a Lock to guarantee they will succeed _lock.acquire() magic_result = magic.detect_from_content(content) _lock.release() mime_type = magic_result.mime_type encoding = magic_result.encoding return mime_type, encoding # maximum authorized content size in bytes for HTML display # with code highlighting content_display_max_size = get_config()['content_display_max_size'] snapshot_content_max_size = get_config()['snapshot_content_max_size'] def _re_encode_content(mimetype, encoding, content_data): # encode textual content to utf-8 if needed if mimetype.startswith('text/'): # probably a malformed UTF-8 content, re-encode it # by replacing invalid chars with a substitution one if encoding == 'unknown-8bit': content_data = content_data.decode('utf-8', 'replace')\ .encode('utf-8') elif encoding not in ['utf-8', 'binary']: content_data = content_data.decode(encoding, 'replace')\ .encode('utf-8') elif mimetype.startswith('application/octet-stream'): # file may detect a text content as binary # so try to decode it for display encodings = ['us-ascii'] encodings += ['iso-8859-%s' % i for i in range(1, 17)] for encoding in encodings: try: content_data = content_data.decode(encoding)\ .encode('utf-8') except Exception: pass else: # ensure display in content view mimetype = 'text/plain' break return mimetype, content_data def request_content(query_string, max_size=content_display_max_size, raise_if_unavailable=True, re_encode=True): """Function that retrieves a content from the archive. Raw bytes content is first retrieved, then the content mime type. If the mime type is not stored in the archive, it will be computed using Python magic module. Args: query_string: a string of the form "[ALGO_HASH:]HASH" where optional ALGO_HASH can be either ``sha1``, ``sha1_git``, ``sha256``, or ``blake2s256`` (default to ``sha1``) and HASH the hexadecimal representation of the hash value max_size: the maximum size for a content to retrieve (default to 1MB, no size limit if None) Returns: A tuple whose first member corresponds to the content raw bytes and second member the content mime type Raises: NotFoundExc if the content is not found """ content_data = service.lookup_content(query_string) filetype = None language = None license = None # requests to the indexer db may fail so properly handle # those cases in order to avoid content display errors try: filetype = service.lookup_content_filetype(query_string) language = service.lookup_content_language(query_string) license = service.lookup_content_license(query_string) except Exception: pass mimetype = 'unknown' encoding = 'unknown' if filetype: mimetype = filetype['mimetype'] encoding = filetype['encoding'] # workaround when encountering corrupted data due to implicit # conversion from bytea to text in the indexer db (see T818) # TODO: Remove that code when all data have been correctly converted if mimetype.startswith('\\'): filetype = None content_data['error_code'] = 200 content_data['error_message'] = '' content_data['error_description'] = '' if not max_size or content_data['length'] < max_size: try: content_raw = service.lookup_content_raw(query_string) except Exception as e: if raise_if_unavailable: raise e else: content_data['raw_data'] = None content_data['error_code'] = 404 content_data['error_description'] = \ 'The bytes of the content are currently not available in the archive.' # noqa content_data['error_message'] = \ http_status_code_message[content_data['error_code']] else: content_data['raw_data'] = content_raw['data'] if not filetype: mimetype, encoding = \ get_mimetype_and_encoding_for_content(content_data['raw_data']) # noqa if re_encode: mimetype, raw_data = _re_encode_content( mimetype, encoding, content_data['raw_data']) content_data['raw_data'] = raw_data else: content_data['raw_data'] = None content_data['mimetype'] = mimetype content_data['encoding'] = encoding if language: content_data['language'] = language['lang'] else: content_data['language'] = 'not detected' if license: content_data['licenses'] = ', '.join(license['facts'][0]['licenses']) else: content_data['licenses'] = 'not detected' return content_data _browsers_supported_image_mimes = set(['image/gif', 'image/png', 'image/jpeg', 'image/bmp', 'image/webp', 'image/svg', 'image/svg+xml']) def prepare_content_for_display(content_data, mime_type, path): """Function that prepares a content for HTML display. The function tries to associate a programming language to a content in order to perform syntax highlighting client-side using highlightjs. The language is determined using either the content filename or its mime type. If the mime type corresponds to an image format supported by web browsers, the content will be encoded in base64 for displaying the image. Args: content_data (bytes): raw bytes of the content mime_type (string): mime type of the content path (string): path of the content including filename Returns: A dict containing the content bytes (possibly different from the one provided as parameter if it is an image) under the key 'content_data and the corresponding highlightjs language class under the key 'language'. """ language = highlightjs.get_hljs_language_from_filename(path) if not language: language = highlightjs.get_hljs_language_from_mime_type(mime_type) if not language: language = 'nohighlight' elif mime_type.startswith('application/'): mime_type = mime_type.replace('application/', 'text/') if mime_type.startswith('image/'): if mime_type in _browsers_supported_image_mimes: content_data = base64.b64encode(content_data) content_data = content_data.decode('utf-8') else: content_data = None if mime_type.startswith('image/svg'): mime_type = 'image/svg+xml' return {'content_data': content_data, 'language': language, 'mimetype': mime_type} def process_snapshot_branches(snapshot): """ Process a dictionary describing snapshot branches: extract those targeting revisions and releases, put them in two different lists, then sort those lists in lexicographical order of the branches' names. Args: snapshot_branches (dict): A dict describing the branches of a snapshot as returned for instance by :func:`swh.web.common.service.lookup_snapshot` Returns: tuple: A tuple whose first member is the sorted list of branches targeting revisions and second member the sorted list of branches targeting releases """ snapshot_branches = snapshot['branches'] branches = {} branch_aliases = {} releases = {} revision_to_branch = defaultdict(set) revision_to_release = defaultdict(set) release_to_branch = defaultdict(set) for branch_name, target in snapshot_branches.items(): if not target: # FIXME: display branches with an unknown target anyway continue target_id = target['target'] target_type = target['target_type'] if target_type == 'revision': branches[branch_name] = { 'name': branch_name, 'revision': target_id, } revision_to_branch[target_id].add(branch_name) elif target_type == 'release': release_to_branch[target_id].add(branch_name) elif target_type == 'alias': branch_aliases[branch_name] = target_id # FIXME: handle pointers to other object types def _enrich_release_branch(branch, release): releases[branch] = { 'name': release['name'], 'branch_name': branch, 'date': format_utc_iso_date(release['date']), 'id': release['id'], 'message': release['message'], 'target_type': release['target_type'], 'target': release['target'], } def _enrich_revision_branch(branch, revision): branches[branch].update({ 'revision': revision['id'], 'directory': revision['directory'], 'date': format_utc_iso_date(revision['date']), 'message': revision['message'] }) releases_info = service.lookup_release_multiple( release_to_branch.keys() ) for release in releases_info: branches_to_update = release_to_branch[release['id']] for branch in branches_to_update: _enrich_release_branch(branch, release) if release['target_type'] == 'revision': revision_to_release[release['target']].update( branches_to_update ) revisions = service.lookup_revision_multiple( set(revision_to_branch.keys()) | set(revision_to_release.keys()) ) for revision in revisions: if not revision: continue for branch in revision_to_branch[revision['id']]: _enrich_revision_branch(branch, revision) for release in revision_to_release[revision['id']]: releases[release]['directory'] = revision['directory'] for branch_alias, branch_target in branch_aliases.items(): if branch_target in branches: branches[branch_alias] = dict(branches[branch_target]) else: snp = service.lookup_snapshot(snapshot['id'], branches_from=branch_target, branches_count=1) if snp and branch_target in snp['branches']: if snp['branches'][branch_target] is None: continue target_type = snp['branches'][branch_target]['target_type'] target = snp['branches'][branch_target]['target'] if target_type == 'revision': branches[branch_alias] = snp['branches'][branch_target] revision = service.lookup_revision(target) _enrich_revision_branch(branch_alias, revision) elif target_type == 'release': release = service.lookup_release(target) _enrich_release_branch(branch_alias, release) if branch_alias in branches: branches[branch_alias]['name'] = branch_alias ret_branches = list(sorted(branches.values(), key=lambda b: b['name'])) ret_releases = list(sorted(releases.values(), key=lambda b: b['name'])) return ret_branches, ret_releases def get_snapshot_content(snapshot_id): """Returns the lists of branches and releases associated to a swh snapshot. That list is put in cache in order to speedup the navigation in the swh-web/browse ui. .. warning:: At most 1000 branches contained in the snapshot will be returned for performance reasons. Args: snapshot_id (str): hexadecimal representation of the snapshot identifier Returns: A tuple with two members. The first one is a list of dict describing the snapshot branches. The second one is a list of dict describing the snapshot releases. Raises: NotFoundExc if the snapshot does not exist """ cache_entry_id = 'swh_snapshot_%s' % snapshot_id cache_entry = cache.get(cache_entry_id) if cache_entry: return cache_entry['branches'], cache_entry['releases'] branches = [] releases = [] if snapshot_id: snapshot = service.lookup_snapshot( snapshot_id, branches_count=snapshot_content_max_size) branches, releases = process_snapshot_branches(snapshot) cache.set(cache_entry_id, { 'branches': branches, 'releases': releases, }) return branches, releases def get_origin_visit_snapshot(origin_info, visit_ts=None, visit_id=None, snapshot_id=None): """Returns the lists of branches and releases associated to a swh origin for a given visit. The visit is expressed by a timestamp. In the latter case, the closest visit from the provided timestamp will be used. If no visit parameter is provided, it returns the list of branches found for the latest visit. That list is put in cache in order to speedup the navigation in the swh-web/browse ui. .. warning:: At most 1000 branches contained in the snapshot will be returned for performance reasons. Args: origin_info (dict): a dict filled with origin information (id, url, type) visit_ts (int or str): an ISO date string or Unix timestamp to parse visit_id (int): optional visit id for disambiguation in case several visits have the same timestamp Returns: A tuple with two members. The first one is a list of dict describing the origin branches for the given visit. The second one is a list of dict describing the origin releases for the given visit. Raises: NotFoundExc if the origin or its visit are not found """ visit_info = get_origin_visit(origin_info, visit_ts, visit_id, snapshot_id) return get_snapshot_content(visit_info['snapshot']) def gen_link(url, link_text=None, link_attrs=None): """ Utility function for generating an HTML link to insert in Django templates. Args: url (str): an url link_text (str): optional text for the produced link, if not provided the url will be used link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ attrs = ' ' if link_attrs: for k, v in link_attrs.items(): attrs += '%s="%s" ' % (k, v) if not link_text: link_text = url link = '%s' \ % (attrs, escape(url), escape(link_text)) return mark_safe(link) def _snapshot_context_query_params(snapshot_context): query_params = None if snapshot_context and snapshot_context['origin_info']: origin_info = snapshot_context['origin_info'] query_params = {'origin': origin_info['url']} if 'timestamp' in snapshot_context['url_args']: query_params['timestamp'] = \ snapshot_context['url_args']['timestamp'] if 'visit_id' in snapshot_context['query_params']: query_params['visit_id'] = \ snapshot_context['query_params']['visit_id'] elif snapshot_context: query_params = {'snapshot_id': snapshot_context['snapshot_id']} return query_params def gen_revision_url(revision_id, snapshot_context=None): """ Utility function for generating an url to a revision. Args: revision_id (str): a revision id snapshot_context (dict): if provided, generate snapshot-dependent browsing url Returns: str: The url to browse the revision """ query_params = _snapshot_context_query_params(snapshot_context) return reverse('browse-revision', url_args={'sha1_git': revision_id}, query_params=query_params) def gen_revision_link(revision_id, shorten_id=False, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a revision HTML view to insert in Django templates. Args: revision_id (str): a revision id shorten_id (boolean): whether to shorten the revision id to 7 characters for the link text snapshot_context (dict): if provided, generate snapshot-dependent browsing link link_text (str): optional text for the generated link (the revision id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: str: An HTML link in the form 'revision_id' """ if not revision_id: return None revision_url = gen_revision_url(revision_id, snapshot_context) if shorten_id: return gen_link(revision_url, revision_id[:7], link_attrs) else: if not link_text: link_text = revision_id return gen_link(revision_url, link_text, link_attrs) def gen_directory_link(sha1_git, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a directory HTML view to insert in Django templates. Args: sha1_git (str): directory identifier link_text (str): optional text for the generated link (the directory id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not sha1_git: return None query_params = _snapshot_context_query_params(snapshot_context) directory_url = reverse('browse-directory', url_args={'sha1_git': sha1_git}, query_params=query_params) if not link_text: link_text = sha1_git return gen_link(directory_url, link_text, link_attrs) def gen_snapshot_link(snapshot_id, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a snapshot HTML view to insert in Django templates. Args: snapshot_id (str): snapshot identifier link_text (str): optional text for the generated link (the snapshot id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ query_params = _snapshot_context_query_params(snapshot_context) snapshot_url = reverse('browse-snapshot', url_args={'snapshot_id': snapshot_id}, query_params=query_params) if not link_text: link_text = snapshot_id return gen_link(snapshot_url, link_text, link_attrs) def gen_content_link(sha1_git, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a content HTML view to insert in Django templates. Args: sha1_git (str): content identifier link_text (str): optional text for the generated link (the content sha1_git will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not sha1_git: return None query_params = _snapshot_context_query_params(snapshot_context) content_url = reverse('browse-content', url_args={'query_string': 'sha1_git:' + sha1_git}, query_params=query_params) if not link_text: link_text = sha1_git return gen_link(content_url, link_text, link_attrs) def get_revision_log_url(revision_id, snapshot_context=None): """ Utility function for getting the URL for a revision log HTML view (possibly in the context of an origin). Args: revision_id (str): revision identifier the history heads to snapshot_context (dict): if provided, generate snapshot-dependent browsing link Returns: The revision log view URL """ query_params = {'revision': revision_id} if snapshot_context and snapshot_context['origin_info']: origin_info = snapshot_context['origin_info'] url_args = {'origin_url': origin_info['url']} if 'timestamp' in snapshot_context['url_args']: url_args['timestamp'] = \ snapshot_context['url_args']['timestamp'] if 'visit_id' in snapshot_context['query_params']: query_params['visit_id'] = \ snapshot_context['query_params']['visit_id'] revision_log_url = reverse('browse-origin-log', url_args=url_args, query_params=query_params) elif snapshot_context: url_args = {'snapshot_id': snapshot_context['snapshot_id']} revision_log_url = reverse('browse-snapshot-log', url_args=url_args, query_params=query_params) else: revision_log_url = reverse('browse-revision-log', url_args={'sha1_git': revision_id}) return revision_log_url def gen_revision_log_link(revision_id, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a revision log HTML view (possibly in the context of an origin) to insert in Django templates. Args: revision_id (str): revision identifier the history heads to snapshot_context (dict): if provided, generate snapshot-dependent browsing link link_text (str): optional text to use for the generated link (the revision id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not revision_id: return None revision_log_url = get_revision_log_url(revision_id, snapshot_context) if not link_text: link_text = revision_id return gen_link(revision_log_url, link_text, link_attrs) def gen_person_mail_link(person, link_text=None): """ Utility function for generating a mail link to a person to insert in Django templates. Args: person (dict): dictionary containing person data (*name*, *email*, *fullname*) link_text (str): optional text to use for the generated mail link (the person name will be used by default) Returns: str: A mail link to the person or the person name if no email is present in person data """ person_name = person['name'] or person['fullname'] if link_text is None: link_text = person_name person_email = person['email'] if person['email'] else None if person_email is None and '@' in person_name and ' ' not in person_name: person_email = person_name if person_email: return gen_link(url='mailto:%s' % person_email, link_text=link_text) else: return person_name def gen_release_link(sha1_git, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a release HTML view to insert in Django templates. Args: sha1_git (str): release identifier link_text (str): optional text for the generated link (the release id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ query_params = _snapshot_context_query_params(snapshot_context) release_url = reverse('browse-release', url_args={'sha1_git': sha1_git}, query_params=query_params) if not link_text: link_text = sha1_git return gen_link(release_url, link_text, link_attrs) def format_log_entries(revision_log, per_page, snapshot_context=None): """ Utility functions that process raw revision log data for HTML display. Its purpose is to: * add links to relevant browse views * format date in human readable format * truncate the message log Args: revision_log (list): raw revision log as returned by the swh-web api per_page (int): number of log entries per page snapshot_context (dict): if provided, generate snapshot-dependent browsing link """ revision_log_data = [] for i, rev in enumerate(revision_log): if i == per_page: break author_name = 'None' author_fullname = 'None' committer_fullname = 'None' if rev['author']: author_name = gen_person_mail_link(rev['author']) author_fullname = rev['author']['fullname'] if rev['committer']: committer_fullname = rev['committer']['fullname'] author_date = format_utc_iso_date(rev['date']) committer_date = format_utc_iso_date(rev['committer_date']) tooltip = 'revision %s\n' % rev['id'] tooltip += 'author: %s\n' % author_fullname tooltip += 'author date: %s\n' % author_date tooltip += 'committer: %s\n' % committer_fullname tooltip += 'committer date: %s\n\n' % committer_date if rev['message']: tooltip += textwrap.indent(rev['message'], ' '*4) revision_log_data.append({ 'author': author_name, 'id': rev['id'][:7], 'message': rev['message'], 'date': author_date, 'commit_date': committer_date, 'url': gen_revision_url(rev['id'], snapshot_context), 'tooltip': tooltip }) return revision_log_data -# list of origin types that can be found in the swh archive -# TODO: retrieve it dynamically in an efficient way instead -# of hardcoding it -_swh_origin_types = ['git', 'svn', 'deb', 'hg', 'ftp', 'deposit', - 'pypi', 'npm'] - - -def get_origin_info(origin_url, origin_type=None): - """ - Get info about a software origin. - Its main purpose is to automatically find an origin type - when it is not provided as parameter. - - Args: - origin_url (str): complete url of a software origin - origin_type (str): optional origin type - - Returns: - A dict with the following entries: - * type: the origin type - * url: the origin url - * id: the internal id of the origin - """ - if origin_type: - return service.lookup_origin({'type': origin_type, - 'url': origin_url}) - else: - for origin_type in _swh_origin_types: - try: - origin_info = service.lookup_origin({'type': origin_type, - 'url': origin_url}) - return origin_info - except Exception: - pass - raise NotFoundExc('Origin with url %s not found!' % escape(origin_url)) - - -def get_snapshot_context(snapshot_id=None, origin_type=None, origin_url=None, +def get_snapshot_context(snapshot_id=None, origin_url=None, timestamp=None, visit_id=None): """ Utility function to compute relevant information when navigating the archive in a snapshot context. The snapshot is either referenced by its id or it will be retrieved from an origin visit. Args: snapshot_id (str): hexadecimal representation of a snapshot identifier, all other parameters will be ignored if it is provided - origin_type (str): the origin type (git, svn, deposit, ...) origin_url (str): the origin_url (e.g. https://github.com/(user)/(repo)/) timestamp (str): a datetime string for retrieving the closest visit of the origin visit_id (int): optional visit id for disambiguation in case of several visits with the same timestamp Returns: A dict with the following entries: * origin_info: dict containing origin information * visit_info: dict containing visit information * branches: the list of branches for the origin found during the visit * releases: the list of releases for the origin found during the visit * origin_browse_url: the url to browse the origin * origin_branches_url: the url to browse the origin branches * origin_releases_url': the url to browse the origin releases * origin_visit_url: the url to browse the snapshot of the origin found during the visit * url_args: dict containing url arguments to use when browsing in the context of the origin and its visit Raises: NotFoundExc: if no snapshot is found for the visit of an origin. """ origin_info = None visit_info = None url_args = None query_params = {} branches = [] releases = [] browse_url = None visit_url = None branches_url = None releases_url = None swh_type = 'snapshot' if origin_url: swh_type = 'origin' - origin_info = get_origin_info(origin_url, origin_type) + origin_info = service.lookup_origin({'url': origin_url}) visit_info = get_origin_visit(origin_info, timestamp, visit_id, snapshot_id) fmt_date = format_utc_iso_date(visit_info['date']) visit_info['fmt_date'] = fmt_date snapshot_id = visit_info['snapshot'] if not snapshot_id: raise NotFoundExc('No snapshot associated to the visit of origin ' '%s on %s' % (escape(origin_url), fmt_date)) # provided timestamp is not necessarily equals to the one # of the retrieved visit, so get the exact one in order # use it in the urls generated below if timestamp: timestamp = visit_info['date'] branches, releases = \ get_origin_visit_snapshot(origin_info, timestamp, visit_id, snapshot_id) - url_args = {'origin_type': origin_type, - 'origin_url': origin_info['url']} + url_args = {'origin_url': origin_info['url']} query_params = {'visit_id': visit_id} browse_url = reverse('browse-origin-visits', url_args=url_args) if timestamp: url_args['timestamp'] = format_utc_iso_date(timestamp, '%Y-%m-%dT%H:%M:%S') visit_url = reverse('browse-origin-directory', url_args=url_args, query_params=query_params) visit_info['url'] = visit_url branches_url = reverse('browse-origin-branches', url_args=url_args, query_params=query_params) releases_url = reverse('browse-origin-releases', url_args=url_args, query_params=query_params) elif snapshot_id: branches, releases = get_snapshot_content(snapshot_id) url_args = {'snapshot_id': snapshot_id} browse_url = reverse('browse-snapshot', url_args=url_args) branches_url = reverse('browse-snapshot-branches', url_args=url_args) releases_url = reverse('browse-snapshot-releases', url_args=url_args) releases = list(reversed(releases)) snapshot_size = service.lookup_snapshot_size(snapshot_id) is_empty = sum(snapshot_size.values()) == 0 swh_snp_id = persistent_identifier('snapshot', snapshot_id) return { 'swh_type': swh_type, 'swh_object_id': swh_snp_id, 'snapshot_id': snapshot_id, 'snapshot_size': snapshot_size, 'is_empty': is_empty, 'origin_info': origin_info, - # keep track if the origin type was provided as url argument - 'origin_type': origin_type, 'visit_info': visit_info, 'branches': branches, 'releases': releases, 'branch': None, 'release': None, 'browse_url': browse_url, 'branches_url': branches_url, 'releases_url': releases_url, 'url_args': url_args, 'query_params': query_params } # list of common readme names ordered by preference # (lower indices have higher priority) _common_readme_names = [ "readme.markdown", "readme.md", "readme.rst", "readme.txt", "readme" ] def get_readme_to_display(readmes): """ Process a list of readme files found in a directory in order to find the adequate one to display. Args: readmes: a list of dict where keys are readme file names and values are readme sha1s Returns: A tuple (readme_name, readme_sha1) """ readme_name = None readme_url = None readme_sha1 = None readme_html = None lc_readmes = {k.lower(): {'orig_name': k, 'sha1': v} for k, v in readmes.items()} # look for readme names according to the preference order # defined by the _common_readme_names list for common_readme_name in _common_readme_names: if common_readme_name in lc_readmes: readme_name = lc_readmes[common_readme_name]['orig_name'] readme_sha1 = lc_readmes[common_readme_name]['sha1'] readme_url = reverse('browse-content-raw', url_args={'query_string': readme_sha1}, query_params={'re_encode': 'true'}) break # otherwise pick the first readme like file if any if not readme_name and len(readmes.items()) > 0: readme_name = next(iter(readmes)) readme_sha1 = readmes[readme_name] readme_url = reverse('browse-content-raw', url_args={'query_string': readme_sha1}, query_params={'re_encode': 'true'}) # convert rst README to html server side as there is # no viable solution to perform that task client side if readme_name and readme_name.endswith('.rst'): cache_entry_id = 'readme_%s' % readme_sha1 cache_entry = cache.get(cache_entry_id) if cache_entry: readme_html = cache_entry else: try: rst_doc = request_content(readme_sha1) readme_html = pypandoc.convert_text(rst_doc['raw_data'], 'html', format='rst') cache.set(cache_entry_id, readme_html) except Exception: readme_html = 'Readme bytes are not available' return readme_name, readme_url, readme_html def get_swh_persistent_ids(swh_objects, snapshot_context=None): """ Returns a list of dict containing info related to persistent identifiers of swh objects. Args: swh_objects (list): a list of dict with the following keys: * type: swh object type (content/directory/release/revision/snapshot) * id: swh object id snapshot_context (dict): optional parameter describing the snapshot in which the object has been found Returns: list: a list of dict with the following keys: * object_type: the swh object type (content/directory/release/revision/snapshot) * object_icon: the swh object icon to use in HTML views * swh_id: the computed swh object persistent identifier * swh_id_url: the url resolving the persistent identifier * show_options: boolean indicating if the persistent id options must be displayed in persistent ids HTML view """ swh_ids = [] for swh_object in swh_objects: if not swh_object['id']: continue swh_id = get_swh_persistent_id(swh_object['type'], swh_object['id']) show_options = swh_object['type'] == 'content' or \ (snapshot_context and snapshot_context['origin_info'] is not None) object_icon = swh_object_icons[swh_object['type']] swh_ids.append({ 'object_type': swh_object['type'], 'object_icon': object_icon, 'swh_id': swh_id, 'swh_id_url': reverse('browse-swh-id', url_args={'swh_id': swh_id}), 'show_options': show_options }) return swh_ids diff --git a/swh/web/browse/views/content.py b/swh/web/browse/views/content.py index f90550b0..37d57511 100644 --- a/swh/web/browse/views/content.py +++ b/swh/web/browse/views/content.py @@ -1,331 +1,329 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import difflib import json from distutils.util import strtobool from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat from swh.model.hashutil import hash_to_hex from swh.web.common import query, service, highlightjs from swh.web.common.utils import ( reverse, gen_path_info, swh_object_icons ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.utils import ( request_content, prepare_content_for_display, content_display_max_size, get_snapshot_context, get_swh_persistent_ids, gen_link, gen_directory_link ) from swh.web.browse.browseurls import browse_route @browse_route(r'content/(?P[0-9a-z_:]*[0-9a-f]+.)/raw/', view_name='browse-content-raw', checksum_args=['query_string']) def content_raw(request, query_string): """Django view that produces a raw display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/raw/` """ try: re_encode = bool(strtobool(request.GET.get('re_encode', 'false'))) algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) content_data = request_content(query_string, max_size=None, re_encode=re_encode) except Exception as exc: return handle_view_exception(request, exc) filename = request.GET.get('filename', None) if not filename: filename = '%s_%s' % (algo, checksum) if content_data['mimetype'].startswith('text/') or \ content_data['mimetype'] == 'inode/x-empty': response = HttpResponse(content_data['raw_data'], content_type="text/plain") response['Content-disposition'] = 'filename=%s' % filename else: response = HttpResponse(content_data['raw_data'], content_type='application/octet-stream') response['Content-disposition'] = 'attachment; filename=%s' % filename return response _auto_diff_size_limit = 20000 @browse_route(r'content/(?P.*)/diff/(?P.*)', # noqa view_name='diff-contents') def _contents_diff(request, from_query_string, to_query_string): """ Browse endpoint used to compute unified diffs between two contents. Diffs are generated only if the two contents are textual. By default, diffs whose size are greater than 20 kB will not be generated. To force the generation of large diffs, the 'force' boolean query parameter must be used. Args: request: input django http request from_query_string: a string of the form "[ALGO_HASH:]HASH" where optional ALGO_HASH can be either ``sha1``, ``sha1_git``, ``sha256``, or ``blake2s256`` (default to ``sha1``) and HASH the hexadecimal representation of the hash value identifying the first content to_query_string: same as above for identifying the second content Returns: A JSON object containing the unified diff. """ diff_data = {} content_from = None content_to = None content_from_size = 0 content_to_size = 0 content_from_lines = [] content_to_lines = [] force = request.GET.get('force', 'false') path = request.GET.get('path', None) language = 'nohighlight' force = bool(strtobool(force)) if from_query_string == to_query_string: diff_str = 'File renamed without changes' else: try: text_diff = True if from_query_string: content_from = \ request_content(from_query_string, max_size=None) content_from_display_data = \ prepare_content_for_display(content_from['raw_data'], content_from['mimetype'], path) language = content_from_display_data['language'] content_from_size = content_from['length'] if not (content_from['mimetype'].startswith('text/') or content_from['mimetype'] == 'inode/x-empty'): text_diff = False if text_diff and to_query_string: content_to = request_content(to_query_string, max_size=None) content_to_display_data = prepare_content_for_display( content_to['raw_data'], content_to['mimetype'], path) language = content_to_display_data['language'] content_to_size = content_to['length'] if not (content_to['mimetype'].startswith('text/') or content_to['mimetype'] == 'inode/x-empty'): text_diff = False diff_size = abs(content_to_size - content_from_size) if not text_diff: diff_str = 'Diffs are not generated for non textual content' language = 'nohighlight' elif not force and diff_size > _auto_diff_size_limit: diff_str = 'Large diffs are not automatically computed' language = 'nohighlight' else: if content_from: content_from_lines = \ content_from['raw_data'].decode('utf-8')\ .splitlines(True) if content_from_lines and \ content_from_lines[-1][-1] != '\n': content_from_lines[-1] += '[swh-no-nl-marker]\n' if content_to: content_to_lines = content_to['raw_data'].decode('utf-8')\ .splitlines(True) if content_to_lines and content_to_lines[-1][-1] != '\n': content_to_lines[-1] += '[swh-no-nl-marker]\n' diff_lines = difflib.unified_diff(content_from_lines, content_to_lines) diff_str = ''.join(list(diff_lines)[2:]) except Exception as e: diff_str = str(e) diff_data['diff_str'] = diff_str diff_data['language'] = language diff_data_json = json.dumps(diff_data, separators=(',', ': ')) return HttpResponse(diff_data_json, content_type='application/json') @browse_route(r'content/(?P[0-9a-z_:]*[0-9a-f]+.)/', view_name='browse-content', checksum_args=['query_string']) def content_display(request, query_string): """Django view that produces an HTML display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/` """ try: algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) content_data = request_content(query_string, raise_if_unavailable=False) - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) selected_language = request.GET.get('language', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: - snapshot_context = get_snapshot_context(None, origin_type, - origin_url) + snapshot_context = get_snapshot_context(origin_url=origin_url) except Exception: raw_cnt_url = reverse('browse-content', url_args={'query_string': query_string}) error_message = \ ('The Software Heritage archive has a content ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the content ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_cnt_url))) raise NotFoundExc(error_message) if snapshot_context: snapshot_context['visit_info'] = None except Exception as exc: return handle_view_exception(request, exc) path = request.GET.get('path', None) content = None language = None mimetype = None if content_data['raw_data'] is not None: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] # Override language with user-selected language if selected_language is not None: language = selected_language available_languages = None if mimetype and 'text/' in mimetype: available_languages = highlightjs.get_supported_languages() root_dir = None filename = None path_info = None directory_id = None directory_url = None query_params = {'origin': origin_url} breadcrumbs = [] if path: split_path = path.split('/') root_dir = split_path[0] filename = split_path[-1] if root_dir != path: path = path.replace(root_dir + '/', '') path = path[:-len(filename)] path_info = gen_path_info(path) dir_url = reverse('browse-directory', url_args={'sha1_git': root_dir}, query_params=query_params) breadcrumbs.append({'name': root_dir[:7], 'url': dir_url}) for pi in path_info: dir_url = reverse('browse-directory', url_args={'sha1_git': root_dir, 'path': pi['path']}, query_params=query_params) breadcrumbs.append({'name': pi['name'], 'url': dir_url}) breadcrumbs.append({'name': filename, 'url': None}) if path and root_dir != path: dir_info = service.lookup_directory_with_path(root_dir, path) directory_id = dir_info['target'] elif root_dir != path: directory_id = root_dir if directory_id: directory_url = gen_directory_link(directory_id) query_params = {'filename': filename} content_raw_url = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params=query_params) content_metadata = { 'sha1': content_data['checksums']['sha1'], 'sha1_git': content_data['checksums']['sha1_git'], 'sha256': content_data['checksums']['sha256'], 'blake2s256': content_data['checksums']['blake2s256'], 'mimetype': content_data['mimetype'], 'encoding': content_data['encoding'], 'size': filesizeformat(content_data['length']), 'language': content_data['language'], 'licenses': content_data['licenses'], 'filename': filename, 'directory': directory_id, 'context-independent directory': directory_url } if filename: content_metadata['filename'] = filename sha1_git = content_data['checksums']['sha1_git'] swh_ids = get_swh_persistent_ids([{'type': 'content', 'id': sha1_git}]) heading = 'Content - %s' % sha1_git if breadcrumbs: content_path = '/'.join([bc['name'] for bc in breadcrumbs]) heading += ' - %s' % content_path return render(request, 'browse/content.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Content', 'swh_object_metadata': content_metadata, 'content': content, 'content_size': content_data['length'], 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'available_languages': available_languages, 'breadcrumbs': breadcrumbs, 'top_right_link': { 'url': content_raw_url, 'icon': swh_object_icons['content'], 'text': 'Raw File' }, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': content_data['error_code'], 'error_message': content_data['error_message'], 'error_description': content_data['error_description']}, status=content_data['error_code']) diff --git a/swh/web/browse/views/directory.py b/swh/web/browse/views/directory.py index 8bc37d2a..fcf58af4 100644 --- a/swh/web/browse/views/directory.py +++ b/swh/web/browse/views/directory.py @@ -1,178 +1,176 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from django.http import HttpResponse from django.shortcuts import render, redirect from django.template.defaultfilters import filesizeformat from swh.web.common import service from swh.web.common.utils import ( reverse, gen_path_info ) from swh.web.common.exc import handle_view_exception, NotFoundExc from swh.web.browse.utils import ( get_directory_entries, get_snapshot_context, get_readme_to_display, get_swh_persistent_ids, gen_link ) from swh.web.browse.browseurls import browse_route @browse_route(r'directory/(?P[0-9a-f]+)/', r'directory/(?P[0-9a-f]+)/(?P.+)/', view_name='browse-directory', checksum_args=['sha1_git']) def directory_browse(request, sha1_git, path=None): """Django view for browsing the content of a directory identified by its sha1_git value. The url that points to it is :http:get:`/browse/directory/(sha1_git)/[(path)/]` """ root_sha1_git = sha1_git try: if path: dir_info = service.lookup_directory_with_path(sha1_git, path) sha1_git = dir_info['target'] dirs, files = get_directory_entries(sha1_git) - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: - snapshot_context = get_snapshot_context(None, origin_type, - origin_url) + snapshot_context = get_snapshot_context(origin_url=origin_url) except Exception: raw_dir_url = reverse('browse-directory', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a directory ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the directory ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_dir_url))) raise NotFoundExc(error_message) if snapshot_context: snapshot_context['visit_info'] = None except Exception as exc: return handle_view_exception(request, exc) path_info = gen_path_info(path) query_params = {'origin': origin_url} breadcrumbs = [] breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse('browse-directory', url_args={'sha1_git': root_sha1_git}, query_params=query_params)}) for pi in path_info: breadcrumbs.append({'name': pi['name'], 'url': reverse('browse-directory', url_args={'sha1_git': root_sha1_git, 'path': pi['path']}, query_params=query_params)}) path = '' if path is None else (path + '/') for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}, query_params=query_params) else: d['url'] = reverse('browse-directory', url_args={'sha1_git': root_sha1_git, 'path': path + d['name']}, query_params=query_params) sum_file_sizes = 0 readmes = {} for f in files: query_string = 'sha1_git:' + f['target'] f['url'] = reverse('browse-content', url_args={'query_string': query_string}, query_params={'path': root_sha1_git + '/' + path + f['name'], 'origin': origin_url}) if f['length'] is not None: sum_file_sizes += f['length'] f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) sum_file_sizes = filesizeformat(sum_file_sizes) dir_metadata = {'directory': sha1_git, 'number of regular files': len(files), 'number of subdirectories': len(dirs), 'sum of regular file sizes': sum_file_sizes} vault_cooking = { 'directory_context': True, 'directory_id': sha1_git, 'revision_context': False, 'revision_id': None } swh_ids = get_swh_persistent_ids([{'type': 'directory', 'id': sha1_git}]) heading = 'Directory - %s' % sha1_git if breadcrumbs: dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' heading += ' - %s' % dir_path return render(request, 'browse/directory.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Directory', 'swh_object_metadata': dir_metadata, 'dirs': dirs, 'files': files, 'breadcrumbs': breadcrumbs, 'top_right_link': None, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'snapshot_context': snapshot_context, 'vault_cooking': vault_cooking, 'show_actions_menu': True, 'swh_ids': swh_ids}) @browse_route(r'directory/resolve/content-path/(?P[0-9a-f]+)/(?P.+)/', # noqa view_name='browse-directory-resolve-content-path', checksum_args=['sha1_git']) def _directory_resolve_content_path(request, sha1_git, path): """ Internal endpoint redirecting to data url for a specific file path relative to a root directory. """ try: path = os.path.normpath(path) if not path.startswith('../'): dir_info = service.lookup_directory_with_path(sha1_git, path) if dir_info['type'] == 'file': sha1 = dir_info['checksums']['sha1'] data_url = reverse('browse-content-raw', url_args={'query_string': sha1}) return redirect(data_url) except Exception: pass return HttpResponse(status=404) diff --git a/swh/web/browse/views/origin.py b/swh/web/browse/views/origin.py index 9dec75ab..d06d556e 100644 --- a/swh/web/browse/views/origin.py +++ b/swh/web/browse/views/origin.py @@ -1,241 +1,208 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from distutils.util import strtobool from django.http import HttpResponse from django.shortcuts import render, redirect from swh.web.common import service from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import ( reverse, format_utc_iso_date, parse_timestamp ) from swh.web.common.exc import handle_view_exception -from swh.web.browse.utils import ( - get_origin_info, get_snapshot_context -) +from swh.web.browse.utils import get_snapshot_context from swh.web.browse.browseurls import browse_route from .utils.snapshot_context import ( browse_snapshot_directory, browse_snapshot_content, browse_snapshot_log, browse_snapshot_branches, browse_snapshot_releases ) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/directory/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/directory/(?P.+)/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/directory/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/directory/(?P.+)/', - r'origin/(?P.+)/visit/(?P.+)/directory/', +@browse_route(r'origin/(?P.+)/visit/(?P.+)/directory/', r'origin/(?P.+)/visit/(?P.+)' '/directory/(?P.+)/', r'origin/(?P.+)/directory/', r'origin/(?P.+)/directory/(?P.+)/', view_name='browse-origin-directory') -def origin_directory_browse(request, origin_url, origin_type=None, +def origin_directory_browse(request, origin_url, timestamp=None, path=None): """Django view for browsing the content of a directory associated to an origin for a given visit. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/]` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/directory/[(path)/]` + * :http:get:`/browse/origin/(origin_url)/directory/[(path)/]` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/directory/[(path)/]` """ # noqa - return browse_snapshot_directory( - request, origin_type=origin_type, origin_url=origin_url, - timestamp=timestamp, path=path) + return browse_snapshot_directory(request, origin_url=origin_url, + timestamp=timestamp, path=path) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/content/(?P.+)/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/content/(?P.+)/', - r'origin/(?P.+)/visit/(?P.+)' +@browse_route(r'origin/(?P.+)/visit/(?P.+)' '/content/(?P.+)/', r'origin/(?P.+)/content/(?P.+)/', view_name='browse-origin-content') -def origin_content_browse(request, origin_url, origin_type=None, path=None, +def origin_content_browse(request, origin_url, path=None, timestamp=None): """Django view that produces an HTML display of a content associated to an origin for a given visit. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/content/(path)/` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/content/(path)/` + * :http:get:`/browse/origin/(origin_url)/content/(path)/` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/content/(path)/` """ # noqa language = request.GET.get('language', None) - return browse_snapshot_content(request, origin_type=origin_type, + return browse_snapshot_content(request, origin_url=origin_url, timestamp=timestamp, path=path, selected_language=language) PER_PAGE = 20 -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/log/', - r'origin/(?P[a-z]+)/url/(?P.+)/log/', - r'origin/(?P.+)/visit/(?P.+)/log/', +@browse_route(r'origin/(?P.+)/visit/(?P.+)/log/', r'origin/(?P.+)/log/', view_name='browse-origin-log') -def origin_log_browse(request, origin_url, origin_type=None, timestamp=None): +def origin_log_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of revisions history (aka the commit log) associated to a software origin. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/log/` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/log/` + * :http:get:`/browse/origin/(origin_url)/log/` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/log/` """ # noqa - return browse_snapshot_log(request, origin_type=origin_type, + return browse_snapshot_log(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/branches/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/branches/', - r'origin/(?P.+)/visit/(?P.+)/branches/', +@browse_route(r'origin/(?P.+)/visit/(?P.+)/branches/', r'origin/(?P.+)/branches/', view_name='browse-origin-branches') -def origin_branches_browse(request, origin_url, origin_type=None, - timestamp=None): +def origin_branches_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of the list of branches associated to an origin for a given visit. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/branches/` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/branches/` + * :http:get:`/browse/origin/(origin_url)/branches/` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/branches/` """ # noqa - return browse_snapshot_branches(request, origin_type=origin_type, + return browse_snapshot_branches(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)' - '/visit/(?P.+)/releases/', - r'origin/(?P[a-z]+)/url/(?P.+)' - '/releases/', - r'origin/(?P.+)/visit/(?P.+)/releases/', +@browse_route(r'origin/(?P.+)/visit/(?P.+)/releases/', r'origin/(?P.+)/releases/', view_name='browse-origin-releases') -def origin_releases_browse(request, origin_url, origin_type=None, - timestamp=None): +def origin_releases_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of the list of releases associated to an origin for a given visit. The url scheme that points to it is the following: - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/releases/` - * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/releases/` + * :http:get:`/browse/origin/(origin_url)/releases/` + * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/releases/` """ # noqa - return browse_snapshot_releases(request, origin_type=origin_type, + return browse_snapshot_releases(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visits/', - r'origin/(?P.+)/visits/', +@browse_route(r'origin/(?P.+)/visits/', view_name='browse-origin-visits') -def origin_visits_browse(request, origin_url, origin_type=None): +def origin_visits_browse(request, origin_url): """Django view that produces an HTML display of visits reporting for a swh origin identified by its id or its url. The url that points to it is - :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visits/`. + :http:get:`/browse/origin/(origin_url)/visits/`. """ try: - origin_info = get_origin_info(origin_url, origin_type) + origin_info = service.lookup_origin({'url': origin_url}) origin_visits = get_origin_visits(origin_info) - snapshot_context = get_snapshot_context(origin_type=origin_type, - origin_url=origin_url) + snapshot_context = get_snapshot_context(origin_url=origin_url) except Exception as exc: return handle_view_exception(request, exc) for i, visit in enumerate(origin_visits): url_date = format_utc_iso_date(visit['date'], '%Y-%m-%dT%H:%M:%SZ') visit['fmt_date'] = format_utc_iso_date(visit['date']) query_params = {} if i < len(origin_visits) - 1: if visit['date'] == origin_visits[i+1]['date']: query_params = {'visit_id': visit['visit']} if i > 0: if visit['date'] == origin_visits[i-1]['date']: query_params = {'visit_id': visit['visit']} snapshot = visit['snapshot'] if visit['snapshot'] else '' visit['browse_url'] = reverse('browse-origin-directory', - url_args={'origin_type': origin_type, - 'origin_url': origin_url, + url_args={'origin_url': origin_url, 'timestamp': url_date}, query_params=query_params) if not snapshot: visit['snapshot'] = '' visit['date'] = parse_timestamp(visit['date']).timestamp() heading = 'Origin visits - %s' % origin_url return render(request, 'browse/origin-visits.html', {'heading': heading, 'swh_object_name': 'Visits', 'swh_object_metadata': origin_info, 'origin_visits': origin_visits, 'origin_info': origin_info, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': False}) @browse_route(r'origin/search/(?P.+)/', view_name='browse-origin-search') def _origin_search(request, url_pattern): """Internal browse endpoint to search for origins whose urls contain a provided string pattern or match a provided regular expression. The search is performed in a case insensitive way. """ offset = int(request.GET.get('offset', '0')) limit = int(request.GET.get('limit', '50')) regexp = request.GET.get('regexp', 'false') with_visit = request.GET.get('with_visit', 'false') url_pattern = url_pattern.replace('///', '\\') try: results = service.search_origin(url_pattern, offset, limit, bool(strtobool(regexp)), bool(strtobool(with_visit))) results = json.dumps(list(results), sort_keys=True, indent=4, separators=(',', ': ')) except Exception as exc: return handle_view_exception(request, exc, html_response=False) return HttpResponse(results, content_type='application/json') -@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/', - r'origin/(?P.+)/', +@browse_route(r'origin/(?P.+)/', view_name='browse-origin') -def origin_browse(request, origin_url, origin_type=None): +def origin_browse(request, origin_url): """Django view that redirects to the display of the latest archived snapshot for a given software origin. """ last_snapshot_url = reverse('browse-origin-directory', - url_args={'origin_type': origin_type, - 'origin_url': origin_url}) + url_args={'origin_url': origin_url}) return redirect(last_snapshot_url) diff --git a/swh/web/browse/views/release.py b/swh/web/browse/views/release.py index d3fba27c..5a402dbe 100644 --- a/swh/web/browse/views/release.py +++ b/swh/web/browse/views/release.py @@ -1,189 +1,185 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import render from swh.web.common import service from swh.web.common.utils import ( reverse, format_utc_iso_date ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.browseurls import browse_route from swh.web.browse.utils import ( gen_revision_link, get_snapshot_context, gen_link, gen_snapshot_link, get_swh_persistent_ids, gen_directory_link, gen_content_link, gen_release_link, gen_person_mail_link ) @browse_route(r'release/(?P[0-9a-f]+)/', view_name='browse-release', checksum_args=['sha1_git']) def release_browse(request, sha1_git): """ Django view that produces an HTML display of a release identified by its id. The url that points to it is :http:get:`/browse/release/(sha1_git)/`. """ try: release = service.lookup_release(sha1_git) snapshot_context = None origin_info = None snapshot_id = request.GET.get('snapshot_id', None) - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) if origin_url: try: - snapshot_context = \ - get_snapshot_context(snapshot_id, origin_type, - origin_url, timestamp, - visit_id) + snapshot_context = get_snapshot_context( + snapshot_id, origin_url, timestamp, visit_id) except Exception: raw_rel_url = reverse('browse-release', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a release ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the release ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_rel_url))) raise NotFoundExc(error_message) origin_info = snapshot_context['origin_info'] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) except Exception as exc: return handle_view_exception(request, exc) release_data = {} release_data['author'] = 'None' if release['author']: release_data['author'] = gen_person_mail_link(release['author']) release_data['date'] = format_utc_iso_date(release['date']) release_data['release'] = sha1_git release_data['name'] = release['name'] release_data['synthetic'] = release['synthetic'] release_data['target'] = release['target'] release_data['target type'] = release['target_type'] if snapshot_context: if release['target_type'] == 'revision': release_data['context-independent target'] = \ gen_revision_link(release['target']) elif release['target_type'] == 'content': release_data['context-independent target'] = \ gen_content_link(release['target']) elif release['target_type'] == 'directory': release_data['context-independent target'] = \ gen_directory_link(release['target']) elif release['target_type'] == 'release': release_data['context-independent target'] = \ gen_release_link(release['target']) release_note_lines = [] if release['message']: release_note_lines = release['message'].split('\n') vault_cooking = None target_link = None if release['target_type'] == 'revision': target_link = gen_revision_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) try: revision = service.lookup_revision(release['target']) vault_cooking = { 'directory_context': True, 'directory_id': revision['directory'], 'revision_context': True, 'revision_id': release['target'] } except Exception: pass elif release['target_type'] == 'directory': target_link = gen_directory_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) try: revision = service.lookup_directory(release['target']) vault_cooking = { 'directory_context': True, 'directory_id': revision['directory'], 'revision_context': False, 'revision_id': None } except Exception: pass elif release['target_type'] == 'content': target_link = gen_content_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) elif release['target_type'] == 'release': target_link = gen_release_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) release['target_link'] = target_link if snapshot_context: release_data['snapshot'] = snapshot_context['snapshot_id'] if origin_info: release_data['context-independent release'] = \ gen_release_link(release['id']) - release_data['origin type'] = origin_info['type'] release_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) browse_snapshot_link = \ gen_snapshot_link(snapshot_context['snapshot_id']) release_data['context-independent snapshot'] = browse_snapshot_link swh_objects = [{'type': 'release', 'id': sha1_git}] if snapshot_context: snapshot_id = snapshot_context['snapshot_id'] if snapshot_id: swh_objects.append({'type': 'snapshot', 'id': snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) note_header = 'None' if len(release_note_lines) > 0: note_header = release_note_lines[0] release['note_header'] = note_header release['note_body'] = '\n'.join(release_note_lines[1:]) heading = 'Release - %s' % release['name'] if snapshot_context: context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading += ' - %s' % context_found return render(request, 'browse/release.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Release', 'swh_object_metadata': release_data, 'release': release, 'snapshot_context': snapshot_context, 'show_actions_menu': True, 'breadcrumbs': None, 'vault_cooking': vault_cooking, 'top_right_link': None, 'swh_ids': swh_ids}) diff --git a/swh/web/browse/views/revision.py b/swh/web/browse/views/revision.py index b2f86a35..76be95e0 100644 --- a/swh/web/browse/views/revision.py +++ b/swh/web/browse/views/revision.py @@ -1,531 +1,525 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import hashlib import json import textwrap from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.utils.html import escape from django.utils.safestring import mark_safe from swh.model.identifiers import persistent_identifier from swh.web.common import service from swh.web.common.utils import ( reverse, format_utc_iso_date, gen_path_info, swh_object_icons ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.browseurls import browse_route from swh.web.browse.utils import ( gen_link, gen_revision_link, gen_revision_url, get_snapshot_context, get_revision_log_url, get_directory_entries, gen_directory_link, request_content, prepare_content_for_display, content_display_max_size, gen_snapshot_link, get_readme_to_display, get_swh_persistent_ids, format_log_entries, gen_person_mail_link ) def _gen_content_url(revision, query_string, path, snapshot_context): if snapshot_context: url_args = snapshot_context['url_args'] url_args['path'] = path query_params = snapshot_context['query_params'] query_params['revision'] = revision['id'] content_url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) else: content_path = '%s/%s' % (revision['directory'], path) content_url = reverse('browse-content', url_args={'query_string': query_string}, query_params={'path': content_path}) return content_url def _gen_diff_link(idx, diff_anchor, link_text): if idx < _max_displayed_file_diffs: return gen_link(diff_anchor, link_text) else: return link_text # TODO: put in conf _max_displayed_file_diffs = 1000 def _gen_revision_changes_list(revision, changes, snapshot_context): """ Returns a HTML string describing the file changes introduced in a revision. As this string will be displayed in the browse revision view, links to adequate file diffs are also generated. Args: revision (str): hexadecimal representation of a revision identifier changes (list): list of file changes in the revision snapshot_context (dict): optional origin context used to reverse the content urls Returns: A string to insert in a revision HTML view. """ changes_msg = [] for i, change in enumerate(changes): hasher = hashlib.sha1() from_query_string = '' to_query_string = '' diff_id = 'diff-' if change['from']: from_query_string = 'sha1_git:' + change['from']['target'] diff_id += change['from']['target'] + '-' + change['from_path'] diff_id += '-' if change['to']: to_query_string = 'sha1_git:' + change['to']['target'] diff_id += change['to']['target'] + change['to_path'] change['path'] = change['to_path'] or change['from_path'] url_args = {'from_query_string': from_query_string, 'to_query_string': to_query_string} query_params = {'path': change['path']} change['diff_url'] = reverse('diff-contents', url_args=url_args, query_params=query_params) hasher.update(diff_id.encode('utf-8')) diff_id = hasher.hexdigest() change['id'] = diff_id panel_diff_link = '#panel_' + diff_id if change['type'] == 'modify': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) changes_msg.append('modified: %s' % _gen_diff_link(i, panel_diff_link, change['to_path'])) elif change['type'] == 'insert': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) changes_msg.append('new file: %s' % _gen_diff_link(i, panel_diff_link, change['to_path'])) elif change['type'] == 'delete': parent = service.lookup_revision(revision['parents'][0]) change['content_url'] = \ _gen_content_url(parent, from_query_string, change['from_path'], snapshot_context) changes_msg.append('deleted: %s' % _gen_diff_link(i, panel_diff_link, change['from_path'])) elif change['type'] == 'rename': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) link_text = change['from_path'] + ' → ' + change['to_path'] changes_msg.append('renamed: %s' % _gen_diff_link(i, panel_diff_link, link_text)) if not changes: changes_msg.append('No changes') return mark_safe('\n'.join(changes_msg)) @browse_route(r'revision/(?P[0-9a-f]+)/diff/', view_name='diff-revision', checksum_args=['sha1_git']) def _revision_diff(request, sha1_git): """ Browse internal endpoint to compute revision diff """ try: revision = service.lookup_revision(sha1_git) snapshot_context = None - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) if origin_url: - snapshot_context = get_snapshot_context(None, origin_type, - origin_url, - timestamp, visit_id) + snapshot_context = get_snapshot_context( + origin_url=origin_url, timestamp=timestamp, visit_id=visit_id) except Exception as exc: return handle_view_exception(request, exc) changes = service.diff_revision(sha1_git) changes_msg = _gen_revision_changes_list(revision, changes, snapshot_context) diff_data = { 'total_nb_changes': len(changes), 'changes': changes[:_max_displayed_file_diffs], 'changes_msg': changes_msg } diff_data_json = json.dumps(diff_data, separators=(',', ': ')) return HttpResponse(diff_data_json, content_type='application/json') NB_LOG_ENTRIES = 100 @browse_route(r'revision/(?P[0-9a-f]+)/log/', view_name='browse-revision-log', checksum_args=['sha1_git']) def revision_log_browse(request, sha1_git): """ Django view that produces an HTML display of the history log for a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/log/` """ try: per_page = int(request.GET.get('per_page', NB_LOG_ENTRIES)) offset = int(request.GET.get('offset', 0)) revs_ordering = request.GET.get('revs_ordering', 'committer_date') session_key = 'rev_%s_log_ordering_%s' % (sha1_git, revs_ordering) rev_log_session = request.session.get(session_key, None) rev_log = [] revs_walker_state = None if rev_log_session: rev_log = rev_log_session['rev_log'] revs_walker_state = rev_log_session['revs_walker_state'] if len(rev_log) < offset+per_page: revs_walker = \ service.get_revisions_walker(revs_ordering, sha1_git, max_revs=offset+per_page+1, state=revs_walker_state) rev_log += [rev['id'] for rev in revs_walker] revs_walker_state = revs_walker.export_state() revs = rev_log[offset:offset+per_page] revision_log = service.lookup_revision_multiple(revs) request.session[session_key] = { 'rev_log': rev_log, 'revs_walker_state': revs_walker_state } except Exception as exc: return handle_view_exception(request, exc) revs_ordering = request.GET.get('revs_ordering', '') prev_log_url = None if len(rev_log) > offset + per_page: prev_log_url = reverse('browse-revision-log', url_args={'sha1_git': sha1_git}, query_params={'per_page': per_page, 'offset': offset + per_page, 'revs_ordering': revs_ordering}) next_log_url = None if offset != 0: next_log_url = reverse('browse-revision-log', url_args={'sha1_git': sha1_git}, query_params={'per_page': per_page, 'offset': offset - per_page, 'revs_ordering': revs_ordering}) revision_log_data = format_log_entries(revision_log, per_page) swh_rev_id = persistent_identifier('revision', sha1_git) return render(request, 'browse/revision-log.html', {'heading': 'Revision history', 'swh_object_id': swh_rev_id, 'swh_object_name': 'Revisions history', 'swh_object_metadata': None, 'revision_log': revision_log_data, 'revs_ordering': revs_ordering, 'next_log_url': next_log_url, 'prev_log_url': prev_log_url, 'breadcrumbs': None, 'top_right_link': None, 'snapshot_context': None, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': None}) @browse_route(r'revision/(?P[0-9a-f]+)/', r'revision/(?P[0-9a-f]+)/(?P.+)/', view_name='browse-revision', checksum_args=['sha1_git']) def revision_browse(request, sha1_git, extra_path=None): """ Django view that produces an HTML display of a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/`. """ try: revision = service.lookup_revision(sha1_git) origin_info = None snapshot_context = None - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) snapshot_id = request.GET.get('snapshot_id', None) path = request.GET.get('path', None) dir_id = None dirs, files = None, None content_data = None if origin_url: try: - snapshot_context = get_snapshot_context(None, origin_type, - origin_url, - timestamp, visit_id) + snapshot_context = get_snapshot_context( + origin_url=origin_url, timestamp=timestamp, + visit_id=visit_id) except Exception: raw_rev_url = reverse('browse-revision', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a revision ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the revision ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_rev_url))) raise NotFoundExc(error_message) origin_info = snapshot_context['origin_info'] snapshot_id = snapshot_context['snapshot_id'] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) if path: file_info = \ service.lookup_directory_with_path(revision['directory'], path) if file_info['type'] == 'dir': dir_id = file_info['target'] else: query_string = 'sha1_git:' + file_info['target'] content_data = request_content(query_string, raise_if_unavailable=False) else: dir_id = revision['directory'] if dir_id: path = '' if path is None else (path + '/') dirs, files = get_directory_entries(dir_id) except Exception as exc: return handle_view_exception(request, exc) revision_data = {} revision_data['author'] = 'None' if revision['author']: author_link = gen_person_mail_link(revision['author']) revision_data['author'] = author_link revision_data['committer'] = 'None' if revision['committer']: committer_link = gen_person_mail_link(revision['committer']) revision_data['committer'] = committer_link revision_data['committer date'] = \ format_utc_iso_date(revision['committer_date']) revision_data['date'] = format_utc_iso_date(revision['date']) revision_data['directory'] = revision['directory'] if snapshot_context: revision_data['snapshot'] = snapshot_id browse_snapshot_link = \ gen_snapshot_link(snapshot_id) revision_data['context-independent snapshot'] = browse_snapshot_link revision_data['context-independent directory'] = \ gen_directory_link(revision['directory']) revision_data['revision'] = sha1_git revision_data['merge'] = revision['merge'] revision_data['metadata'] = escape(json.dumps(revision['metadata'], sort_keys=True, indent=4, separators=(',', ': '))) if origin_info: - revision_data['origin type'] = origin_info['type'] revision_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) revision_data['context-independent revision'] = \ gen_revision_link(sha1_git) parents = '' for p in revision['parents']: parent_link = gen_revision_link(p, link_text=None, link_attrs=None, snapshot_context=snapshot_context) parents += parent_link + '
' revision_data['parents'] = mark_safe(parents) revision_data['synthetic'] = revision['synthetic'] revision_data['type'] = revision['type'] message_lines = ['None'] if revision['message']: message_lines = revision['message'].split('\n') parents = [] for p in revision['parents']: parent_url = gen_revision_url(p, snapshot_context) parents.append({'id': p, 'url': parent_url}) path_info = gen_path_info(path) query_params = {'snapshot_id': snapshot_id, - 'origin_type': origin_type, 'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id} breadcrumbs = [] breadcrumbs.append({'name': revision['directory'][:7], 'url': reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params)}) for pi in path_info: query_params['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params)}) vault_cooking = { 'directory_context': False, 'directory_id': None, 'revision_context': True, 'revision_id': sha1_git } swh_objects = [{'type': 'revision', 'id': sha1_git}] content = None content_size = None mimetype = None language = None readme_name = None readme_url = None readme_html = None readmes = {} error_code = 200 error_message = '' error_description = '' if content_data: breadcrumbs[-1]['url'] = None content_size = content_data['length'] mimetype = content_data['mimetype'] if content_data['raw_data']: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] query_params = {} if path: filename = path_info[-1]['name'] query_params['filename'] = path_info[-1]['name'] revision_data['filename'] = filename top_right_link = { 'url': reverse('browse-content-raw', url_args={'query_string': query_string}, query_params=query_params), 'icon': swh_object_icons['content'], 'text': 'Raw File' } swh_objects.append({'type': 'content', 'id': file_info['target']}) error_code = content_data['error_code'] error_message = content_data['error_message'] error_description = content_data['error_description'] else: for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: query_params['path'] = path + d['name'] d['url'] = reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) for f in files: query_params['path'] = path + f['name'] f['url'] = reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) if f['length'] is not None: f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) top_right_link = { 'url': get_revision_log_url(sha1_git, snapshot_context), 'icon': swh_object_icons['revisions history'], 'text': 'History' } vault_cooking['directory_context'] = True vault_cooking['directory_id'] = dir_id swh_objects.append({'type': 'directory', 'id': dir_id}) diff_revision_url = reverse('diff-revision', url_args={'sha1_git': sha1_git}, - query_params={'origin_type': origin_type, - 'origin': origin_url, + query_params={'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id}) if snapshot_id: swh_objects.append({'type': 'snapshot', 'id': snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) heading = 'Revision - %s - %s' %\ (sha1_git[:7], textwrap.shorten(message_lines[0], width=70)) if snapshot_context: context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading += ' - %s' % context_found return render(request, 'browse/revision.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Revision', 'swh_object_metadata': revision_data, 'message_header': message_lines[0], 'message_body': '\n'.join(message_lines[1:]), 'parents': parents, 'snapshot_context': snapshot_context, 'dirs': dirs, 'files': files, 'content': content, 'content_size': content_size, 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'breadcrumbs': breadcrumbs, 'top_right_link': top_right_link, 'vault_cooking': vault_cooking, 'diff_revision_url': diff_revision_url, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': error_code, 'error_message': error_message, 'error_description': error_description}, status=error_code) diff --git a/swh/web/browse/views/snapshot.py b/swh/web/browse/views/snapshot.py index f42fe399..de113b8f 100644 --- a/swh/web/browse/views/snapshot.py +++ b/swh/web/browse/views/snapshot.py @@ -1,104 +1,102 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import redirect from swh.web.browse.browseurls import browse_route from swh.web.common.utils import reverse from .utils.snapshot_context import ( browse_snapshot_directory, browse_snapshot_content, browse_snapshot_log, browse_snapshot_branches, browse_snapshot_releases ) @browse_route(r'snapshot/(?P[0-9a-f]+)/', view_name='browse-snapshot', checksum_args=['snapshot_id']) def snapshot_browse(request, snapshot_id): """Django view for browsing the content of a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/` """ browse_snapshot_url = reverse('browse-snapshot-directory', url_args={'snapshot_id': snapshot_id}, query_params=request.GET) return redirect(browse_snapshot_url) @browse_route(r'snapshot/(?P[0-9a-f]+)/directory/', r'snapshot/(?P[0-9a-f]+)/directory/(?P.+)/', view_name='browse-snapshot-directory', checksum_args=['snapshot_id']) def snapshot_directory_browse(request, snapshot_id, path=None): """Django view for browsing the content of a directory collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/directory/[(path)/]` """ - origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) return browse_snapshot_directory(request, snapshot_id=snapshot_id, - path=path, origin_type=origin_type, - origin_url=origin_url) + path=path, origin_url=origin_url) @browse_route(r'snapshot/(?P[0-9a-f]+)/content/(?P.+)/', view_name='browse-snapshot-content', checksum_args=['snapshot_id']) def snapshot_content_browse(request, snapshot_id, path): """Django view that produces an HTML display of a content collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/content/(path)/` """ language = request.GET.get('language', None) return browse_snapshot_content(request, snapshot_id=snapshot_id, path=path, selected_language=language) @browse_route(r'snapshot/(?P[0-9a-f]+)/log/', view_name='browse-snapshot-log', checksum_args=['snapshot_id']) def snapshot_log_browse(request, snapshot_id): """Django view that produces an HTML display of revisions history (aka the commit log) collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/log/` """ return browse_snapshot_log(request, snapshot_id=snapshot_id) @browse_route(r'snapshot/(?P[0-9a-f]+)/branches/', view_name='browse-snapshot-branches', checksum_args=['snapshot_id']) def snapshot_branches_browse(request, snapshot_id): """Django view that produces an HTML display of the list of releases collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/branches/` """ return browse_snapshot_branches(request, snapshot_id=snapshot_id) @browse_route(r'snapshot/(?P[0-9a-f]+)/releases/', view_name='browse-snapshot-releases', checksum_args=['snapshot_id']) def snapshot_releases_browse(request, snapshot_id): """Django view that produces an HTML display of the list of releases collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/releases/` """ return browse_snapshot_releases(request, snapshot_id=snapshot_id) diff --git a/swh/web/browse/views/utils/snapshot_context.py b/swh/web/browse/views/utils/snapshot_context.py index 402cae30..7a4bd297 100644 --- a/swh/web/browse/views/utils/snapshot_context.py +++ b/swh/web/browse/views/utils/snapshot_context.py @@ -1,903 +1,898 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information # Utility module implementing Django views for browsing the archive # in a snapshot context. # Its purpose is to factorize code for the views reachable from the # /origin/.* and /snapshot/.* endpoints. from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.utils.html import escape from swh.model.identifiers import snapshot_identifier from swh.web.browse.utils import ( get_snapshot_context, get_directory_entries, gen_directory_link, gen_revision_link, request_content, gen_content_link, prepare_content_for_display, content_display_max_size, format_log_entries, gen_revision_log_link, get_readme_to_display, get_swh_persistent_ids, gen_snapshot_link, process_snapshot_branches ) from swh.web.common import service, highlightjs from swh.web.common.exc import ( handle_view_exception, NotFoundExc ) from swh.web.common.utils import ( reverse, gen_path_info, format_utc_iso_date, swh_object_icons ) _empty_snapshot_id = snapshot_identifier({'branches': {}}) def _get_branch(branches, branch_name, snapshot_id): """ Utility function to get a specific branch from a branches list. Its purpose is to get the default HEAD branch as some software origin (e.g those with svn type) does not have it. In that latter case, check if there is a master branch instead and returns it. """ filtered_branches = [b for b in branches if b['name'] == branch_name] if filtered_branches: return filtered_branches[0] elif branch_name == 'HEAD': filtered_branches = [b for b in branches if b['name'].endswith('master')] if filtered_branches: return filtered_branches[0] elif branches: return branches[0] else: # case where a large branches list has been truncated snp = service.lookup_snapshot(snapshot_id, branches_from=branch_name, branches_count=1, target_types=['revision', 'alias']) snp_branch, _ = process_snapshot_branches(snp) if snp_branch and snp_branch[0]['name'] == branch_name: branches.append(snp_branch[0]) return snp_branch[0] def _get_release(releases, release_name): """ Utility function to get a specific release from a releases list. Returns None if the release can not be found in the list. """ filtered_releases = [r for r in releases if r['name'] == release_name] if filtered_releases: return filtered_releases[0] def _branch_not_found(branch_type, branch, branches, snapshot_id=None, origin_info=None, timestamp=None, visit_id=None): """ Utility function to raise an exception when a specified branch/release can not be found. """ if branch_type == 'branch': branch_type = 'Branch' branch_type_plural = 'branches' else: branch_type = 'Release' branch_type_plural = 'releases' if snapshot_id and not branches: msg = ('Snapshot with id %s has an empty list' ' of %s!' % (snapshot_id, branch_type_plural)) elif snapshot_id: msg = ('%s %s for snapshot with id %s' ' not found!' % (branch_type, branch, snapshot_id)) elif visit_id and not branches: - msg = ('Origin with type %s and url %s' + msg = ('Origin with url %s' ' for visit with id %s has an empty list' - ' of %s!' % (origin_info['type'], origin_info['url'], visit_id, + ' of %s!' % (origin_info['url'], visit_id, branch_type_plural)) elif visit_id: msg = ('%s %s associated to visit with' - ' id %s for origin with type %s and url %s' + ' id %s for origin with url %s' ' not found!' % (branch_type, branch, visit_id, - origin_info['type'], origin_info['url'])) + origin_info['url'])) elif not branches: - msg = ('Origin with type %s and url %s' + msg = ('Origin with url %s' ' for visit with timestamp %s has an empty list' - ' of %s!' % (origin_info['type'], origin_info['url'], + ' of %s!' % (origin_info['url'], timestamp, branch_type_plural)) else: msg = ('%s %s associated to visit with' - ' timestamp %s for origin with type %s' - ' and url %s not found!' % (branch_type, branch, timestamp, - origin_info['type'], - origin_info['url'])) + ' timestamp %s for origin with ' + 'url %s not found!' % (branch_type, branch, timestamp, + origin_info['url'])) raise NotFoundExc(escape(msg)) -def _process_snapshot_request(request, snapshot_id=None, origin_type=None, +def _process_snapshot_request(request, snapshot_id=None, origin_url=None, timestamp=None, path=None, browse_context='directory'): """ Utility function to perform common input request processing for snapshot context views. """ visit_id = request.GET.get('visit_id', None) - snapshot_context = get_snapshot_context(snapshot_id, origin_type, + snapshot_context = get_snapshot_context(snapshot_id, origin_url, timestamp, visit_id) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] branches = snapshot_context['branches'] releases = snapshot_context['releases'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] if snapshot_context['visit_info']: timestamp = format_utc_iso_date(snapshot_context['visit_info']['date'], '%Y-%m-%dT%H:%M:%SZ') snapshot_context['timestamp'] = format_utc_iso_date( snapshot_context['visit_info']['date']) browse_view_name = 'browse-' + swh_type + '-' + browse_context root_sha1_git = None revision_id = request.GET.get('revision', None) release_name = request.GET.get('release', None) release_id = None branch_name = None snapshot_total_size = sum(snapshot_context['snapshot_size'].values()) if snapshot_total_size and revision_id: revision = service.lookup_revision(revision_id) root_sha1_git = revision['directory'] branches.append({'name': revision_id, 'revision': revision_id, 'directory': root_sha1_git, 'url': None}) branch_name = revision_id query_params['revision'] = revision_id elif snapshot_total_size and release_name: release = _get_release(releases, release_name) try: root_sha1_git = release['directory'] revision_id = release['target'] release_id = release['id'] query_params['release'] = release_name except Exception: _branch_not_found("release", release_name, releases, snapshot_id, origin_info, timestamp, visit_id) elif snapshot_total_size: branch_name = request.GET.get('branch', None) if branch_name: query_params['branch'] = branch_name branch = _get_branch(branches, branch_name or 'HEAD', snapshot_context['snapshot_id']) try: branch_name = branch['name'] revision_id = branch['revision'] root_sha1_git = branch['directory'] except Exception: _branch_not_found("branch", branch_name, branches, snapshot_id, origin_info, timestamp, visit_id) for b in branches: branch_url_args = dict(url_args) branch_query_params = dict(query_params) if 'release' in branch_query_params: del branch_query_params['release'] branch_query_params['branch'] = b['name'] if path: b['path'] = path branch_url_args['path'] = path b['url'] = reverse(browse_view_name, url_args=branch_url_args, query_params=branch_query_params) for r in releases: release_url_args = dict(url_args) release_query_params = dict(query_params) if 'branch' in release_query_params: del release_query_params['branch'] release_query_params['release'] = r['name'] if path: r['path'] = path release_url_args['path'] = path r['url'] = reverse(browse_view_name, url_args=release_url_args, query_params=release_query_params) snapshot_context['query_params'] = query_params snapshot_context['root_sha1_git'] = root_sha1_git snapshot_context['revision_id'] = revision_id snapshot_context['branch'] = branch_name snapshot_context['release'] = release_name snapshot_context['release_id'] = release_id return snapshot_context -def browse_snapshot_directory(request, snapshot_id=None, origin_type=None, +def browse_snapshot_directory(request, snapshot_id=None, origin_url=None, timestamp=None, path=None): """ Django view implementation for browsing a directory in a snapshot context. """ try: snapshot_context = _process_snapshot_request( - request, snapshot_id, origin_type, origin_url, + request, snapshot_id, origin_url, timestamp, path, browse_context='directory') root_sha1_git = snapshot_context['root_sha1_git'] sha1_git = root_sha1_git if root_sha1_git and path: dir_info = service.lookup_directory_with_path(root_sha1_git, path) sha1_git = dir_info['target'] dirs = [] files = [] if sha1_git: dirs, files = get_directory_entries(sha1_git) except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] revision_id = snapshot_context['revision_id'] snapshot_id = snapshot_context['snapshot_id'] path_info = gen_path_info(path) browse_view_name = 'browse-' + swh_type + '-directory' breadcrumbs = [] if root_sha1_git: breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse(browse_view_name, url_args=url_args, query_params=query_params)}) for pi in path_info: bc_url_args = dict(url_args) bc_url_args['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse(browse_view_name, url_args=bc_url_args, query_params=query_params)}) path = '' if path is None else (path + '/') for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: bc_url_args = dict(url_args) bc_url_args['path'] = path + d['name'] d['url'] = reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) sum_file_sizes = 0 readmes = {} browse_view_name = 'browse-' + swh_type + '-content' for f in files: bc_url_args = dict(url_args) bc_url_args['path'] = path + f['name'] f['url'] = reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) if f['length'] is not None: sum_file_sizes += f['length'] f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) browse_view_name = 'browse-' + swh_type + '-log' history_url = None if snapshot_id != _empty_snapshot_id: history_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) nb_files = None nb_dirs = None dir_path = None if root_sha1_git: nb_files = len(files) nb_dirs = len(dirs) sum_file_sizes = filesizeformat(sum_file_sizes) dir_path = '/' + path browse_dir_link = gen_directory_link(sha1_git) browse_rev_link = gen_revision_link(revision_id) browse_snp_link = gen_snapshot_link(snapshot_id) dir_metadata = {'directory': sha1_git, 'context-independent directory': browse_dir_link, 'number of regular files': nb_files, 'number of subdirectories': nb_dirs, 'sum of regular file sizes': sum_file_sizes, 'path': dir_path, 'revision': revision_id, 'context-independent revision': browse_rev_link, 'snapshot': snapshot_id, 'context-independent snapshot': browse_snp_link} if origin_info: - dir_metadata['origin type'] = origin_info['type'] dir_metadata['origin url'] = origin_info['url'] dir_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) + dir_metadata['origin visit type'] = visit_info['type'] vault_cooking = { 'directory_context': True, 'directory_id': sha1_git, 'revision_context': True, 'revision_id': revision_id } swh_objects = [{'type': 'directory', 'id': sha1_git}, {'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = ('Directory - %s - %s - %s' % (dir_path, snapshot_context['branch'], context_found)) return render(request, 'browse/directory.html', {'heading': heading, 'swh_object_name': 'Directory', 'swh_object_metadata': dir_metadata, 'dirs': dirs, 'files': files, 'breadcrumbs': breadcrumbs if root_sha1_git else [], 'top_right_link': { 'url': history_url, 'icon': swh_object_icons['revisions history'], 'text': 'History' }, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'snapshot_context': snapshot_context, 'vault_cooking': vault_cooking, 'show_actions_menu': True, 'swh_ids': swh_ids}) -def browse_snapshot_content(request, snapshot_id=None, origin_type=None, +def browse_snapshot_content(request, snapshot_id=None, origin_url=None, timestamp=None, path=None, selected_language=None): """ Django view implementation for browsing a content in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_type, origin_url, + origin_url, timestamp, path, browse_context='content') root_sha1_git = snapshot_context['root_sha1_git'] sha1_git = None query_string = None content_data = None split_path = path.split('/') filename = split_path[-1] filepath = path[:-len(filename)] if root_sha1_git: content_info = service.lookup_directory_with_path(root_sha1_git, path) sha1_git = content_info['target'] query_string = 'sha1_git:' + sha1_git content_data = request_content(query_string, raise_if_unavailable=False) if filepath: dir_info = service.lookup_directory_with_path(root_sha1_git, filepath) directory_id = dir_info['target'] else: directory_id = root_sha1_git except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] revision_id = snapshot_context['revision_id'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] snapshot_id = snapshot_context['snapshot_id'] content = None language = None mimetype = None if content_data and content_data['raw_data'] is not None: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] # Override language with user-selected language if selected_language is not None: language = selected_language available_languages = None if mimetype and 'text/' in mimetype: available_languages = highlightjs.get_supported_languages() browse_view_name = 'browse-' + swh_type + '-directory' breadcrumbs = [] path_info = gen_path_info(filepath) if root_sha1_git: breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse(browse_view_name, url_args=url_args, query_params=query_params)}) for pi in path_info: bc_url_args = dict(url_args) bc_url_args['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse(browse_view_name, url_args=bc_url_args, query_params=query_params)}) breadcrumbs.append({'name': filename, 'url': None}) browse_content_link = gen_content_link(sha1_git) content_raw_url = None if query_string: content_raw_url = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params={'filename': filename}) browse_rev_link = gen_revision_link(revision_id) browse_dir_link = gen_directory_link(directory_id) content_metadata = { 'context-independent content': browse_content_link, 'path': None, 'filename': None, 'directory': directory_id, 'context-independent directory': browse_dir_link, 'revision': revision_id, 'context-independent revision': browse_rev_link, 'snapshot': snapshot_id } cnt_sha1_git = None content_size = None error_code = 200 error_description = '' error_message = '' if content_data: for checksum in content_data['checksums'].keys(): content_metadata[checksum] = content_data['checksums'][checksum] content_metadata['mimetype'] = content_data['mimetype'] content_metadata['encoding'] = content_data['encoding'] content_metadata['size'] = filesizeformat(content_data['length']) content_metadata['language'] = content_data['language'] content_metadata['licenses'] = content_data['licenses'] content_metadata['path'] = '/' + filepath content_metadata['filename'] = filename cnt_sha1_git = content_data['checksums']['sha1_git'] content_size = content_data['length'] error_code = content_data['error_code'] error_message = content_data['error_message'] error_description = content_data['error_description'] if origin_info: - content_metadata['origin type'] = origin_info['type'] content_metadata['origin url'] = origin_info['url'] content_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) + content_metadata['origin visit type'] = visit_info['type'] browse_snapshot_link = gen_snapshot_link(snapshot_id) content_metadata['context-independent snapshot'] = browse_snapshot_link swh_objects = [{'type': 'content', 'id': cnt_sha1_git}, {'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) content_path = '/'.join([bc['name'] for bc in breadcrumbs]) context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = ('Content - %s - %s - %s' % (content_path, snapshot_context['branch'], context_found)) return render(request, 'browse/content.html', {'heading': heading, 'swh_object_name': 'Content', 'swh_object_metadata': content_metadata, 'content': content, 'content_size': content_size, 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'available_languages': available_languages, 'breadcrumbs': breadcrumbs if root_sha1_git else [], 'top_right_link': { 'url': content_raw_url, 'icon': swh_object_icons['content'], 'text': 'Raw File' }, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': error_code, 'error_message': error_message, 'error_description': error_description}, status=error_code) PER_PAGE = 100 -def browse_snapshot_log(request, snapshot_id=None, origin_type=None, +def browse_snapshot_log(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a revision history in a snapshot context. """ try: snapshot_context = _process_snapshot_request( - request, snapshot_id, origin_type, origin_url, - timestamp, browse_context='log') + request, snapshot_id, origin_url, timestamp, browse_context='log') revision_id = snapshot_context['revision_id'] per_page = int(request.GET.get('per_page', PER_PAGE)) offset = int(request.GET.get('offset', 0)) revs_ordering = request.GET.get('revs_ordering', 'committer_date') session_key = 'rev_%s_log_ordering_%s' % (revision_id, revs_ordering) rev_log_session = request.session.get(session_key, None) rev_log = [] revs_walker_state = None if rev_log_session: rev_log = rev_log_session['rev_log'] revs_walker_state = rev_log_session['revs_walker_state'] if len(rev_log) < offset+per_page: revs_walker = service.get_revisions_walker( revs_ordering, revision_id, max_revs=offset+per_page+1, state=revs_walker_state) rev_log += [rev['id'] for rev in revs_walker] revs_walker_state = revs_walker.export_state() revs = rev_log[offset:offset+per_page] revision_log = service.lookup_revision_multiple(revs) request.session[session_key] = { 'rev_log': rev_log, 'revs_walker_state': revs_walker_state } except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] snapshot_id = snapshot_context['snapshot_id'] query_params['per_page'] = per_page revs_ordering = request.GET.get('revs_ordering', '') query_params['revs_ordering'] = revs_ordering browse_view_name = 'browse-' + swh_type + '-log' prev_log_url = None if len(rev_log) > offset + per_page: query_params['offset'] = offset + per_page prev_log_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) next_log_url = None if offset != 0: query_params['offset'] = offset - per_page next_log_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) revision_log_data = format_log_entries(revision_log, per_page, snapshot_context) browse_rev_link = gen_revision_link(revision_id) browse_log_link = gen_revision_log_link(revision_id) browse_snp_link = gen_snapshot_link(snapshot_id) revision_metadata = { 'context-independent revision': browse_rev_link, 'context-independent revision history': browse_log_link, 'context-independent snapshot': browse_snp_link, 'snapshot': snapshot_id } if origin_info: - revision_metadata['origin type'] = origin_info['type'] revision_metadata['origin url'] = origin_info['url'] revision_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) + revision_metadata['origin visit type'] = visit_info['type'] swh_objects = [{'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = ('Revision history - %s - %s' % (snapshot_context['branch'], context_found)) return render(request, 'browse/revision-log.html', {'heading': heading, 'swh_object_name': 'Revisions history', 'swh_object_metadata': revision_metadata, 'revision_log': revision_log_data, 'revs_ordering': revs_ordering, 'next_log_url': next_log_url, 'prev_log_url': prev_log_url, 'breadcrumbs': None, 'top_right_link': None, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids}) -def browse_snapshot_branches(request, snapshot_id=None, origin_type=None, +def browse_snapshot_branches(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a list of branches in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_type, origin_url, - timestamp) + origin_url, timestamp) branches_bc = request.GET.get('branches_breadcrumbs', '') branches_bc = branches_bc.split(',') if branches_bc else [] branches_from = branches_bc[-1] if branches_bc else '' swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] browse_view_name = 'browse-' + swh_type + '-directory' snapshot = service.lookup_snapshot(snapshot_context['snapshot_id'], branches_from, PER_PAGE+1, target_types=['revision', 'alias']) displayed_branches, _ = process_snapshot_branches(snapshot) except Exception as exc: return handle_view_exception(request, exc) for branch in displayed_branches: if snapshot_id: revision_url = reverse('browse-revision', url_args={'sha1_git': branch['revision']}, query_params={'snapshot_id': snapshot_id}) else: revision_url = reverse('browse-revision', url_args={'sha1_git': branch['revision']}, - query_params={'origin_type': origin_type, - 'origin': origin_info['url']}) + query_params={'origin': origin_info['url']}) query_params['branch'] = branch['name'] directory_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) del query_params['branch'] branch['revision_url'] = revision_url branch['directory_url'] = directory_url browse_view_name = 'browse-' + swh_type + '-branches' prev_branches_url = None next_branches_url = None if branches_bc: query_params_prev = dict(query_params) query_params_prev['branches_breadcrumbs'] = ','.join(branches_bc[:-1]) prev_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_prev) elif branches_from: prev_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) if len(displayed_branches) > PER_PAGE: query_params_next = dict(query_params) next_branch = displayed_branches[-1]['name'] del displayed_branches[-1] branches_bc.append(next_branch) query_params_next['branches_breadcrumbs'] = ','.join(branches_bc) next_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_next) heading = 'Branches - ' if origin_info: heading += 'origin: %s' % origin_info['url'] else: heading += 'snapshot: %s' % snapshot_id return render(request, 'browse/branches.html', {'heading': heading, 'swh_object_name': 'Branches', 'swh_object_metadata': {}, 'top_right_link': None, 'displayed_branches': displayed_branches, 'prev_branches_url': prev_branches_url, 'next_branches_url': next_branches_url, 'snapshot_context': snapshot_context}) -def browse_snapshot_releases(request, snapshot_id=None, origin_type=None, +def browse_snapshot_releases(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a list of releases in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_type, origin_url, - timestamp) + origin_url, timestamp) rel_bc = request.GET.get('releases_breadcrumbs', '') rel_bc = rel_bc.split(',') if rel_bc else [] rel_from = rel_bc[-1] if rel_bc else '' swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] snapshot = service.lookup_snapshot(snapshot_context['snapshot_id'], rel_from, PER_PAGE+1, target_types=['release', 'alias']) _, displayed_releases = process_snapshot_branches(snapshot) except Exception as exc: return handle_view_exception(request, exc) for release in displayed_releases: if snapshot_id: query_params_tgt = {'snapshot_id': snapshot_id} else: query_params_tgt = {'origin': origin_info['url']} release_url = reverse('browse-release', url_args={'sha1_git': release['id']}, query_params=query_params_tgt) target_url = '' if release['target_type'] == 'revision': target_url = reverse('browse-revision', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'directory': target_url = reverse('browse-directory', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'content': target_url = reverse('browse-content', url_args={'query_string': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'release': target_url = reverse('browse-release', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) release['release_url'] = release_url release['target_url'] = target_url browse_view_name = 'browse-' + swh_type + '-releases' prev_releases_url = None next_releases_url = None if rel_bc: query_params_prev = dict(query_params) query_params_prev['releases_breadcrumbs'] = ','.join(rel_bc[:-1]) prev_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_prev) elif rel_from: prev_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) if len(displayed_releases) > PER_PAGE: query_params_next = dict(query_params) next_rel = displayed_releases[-1]['branch_name'] del displayed_releases[-1] rel_bc.append(next_rel) query_params_next['releases_breadcrumbs'] = ','.join(rel_bc) next_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_next) heading = 'Releases - ' if origin_info: heading += 'origin: %s' % origin_info['url'] else: heading += 'snapshot: %s' % snapshot_id return render(request, 'browse/releases.html', {'heading': heading, 'top_panel_visible': False, 'top_panel_collapsible': False, 'swh_object_name': 'Releases', 'swh_object_metadata': {}, 'top_right_link': None, 'displayed_releases': displayed_releases, 'prev_releases_url': prev_releases_url, 'next_releases_url': next_releases_url, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': False}) diff --git a/swh/web/common/migrations/0006_rename_origin_type.py b/swh/web/common/migrations/0006_rename_origin_type.py new file mode 100644 index 00000000..017b9e9e --- /dev/null +++ b/swh/web/common/migrations/0006_rename_origin_type.py @@ -0,0 +1,23 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('swh.web.common', '0005_remove_duplicated_authorized_origins'), + ] + + operations = [ + migrations.RenameField( + model_name='saveoriginrequest', + old_name='origin_type', + new_name='visit_type', + ), + ] diff --git a/swh/web/common/models.py b/swh/web/common/models.py index cfffb784..f0b66c5f 100644 --- a/swh/web/common/models.py +++ b/swh/web/common/models.py @@ -1,92 +1,92 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.db import models class SaveAuthorizedOrigin(models.Model): """ Model table holding origin urls authorized to be loaded into the archive. """ url = models.CharField(max_length=200, null=False) class Meta: app_label = 'swh.web.common' db_table = 'save_authorized_origin' def __str__(self): return self.url class SaveUnauthorizedOrigin(models.Model): """ Model table holding origin urls not authorized to be loaded into the archive. """ url = models.CharField(max_length=200, null=False) class Meta: app_label = 'swh.web.common' db_table = 'save_unauthorized_origin' def __str__(self): return self.url SAVE_REQUEST_ACCEPTED = 'accepted' SAVE_REQUEST_REJECTED = 'rejected' SAVE_REQUEST_PENDING = 'pending' SAVE_REQUEST_STATUS = [ (SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_ACCEPTED), (SAVE_REQUEST_REJECTED, SAVE_REQUEST_REJECTED), (SAVE_REQUEST_PENDING, SAVE_REQUEST_PENDING) ] SAVE_TASK_NOT_CREATED = 'not created' SAVE_TASK_NOT_YET_SCHEDULED = 'not yet scheduled' SAVE_TASK_SCHEDULED = 'scheduled' SAVE_TASK_SUCCEED = 'succeed' SAVE_TASK_FAILED = 'failed' SAVE_TASK_RUNNING = 'running' SAVE_TASK_STATUS = [ (SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_CREATED), (SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED), (SAVE_TASK_SCHEDULED, SAVE_TASK_SCHEDULED), (SAVE_TASK_SUCCEED, SAVE_TASK_SUCCEED), (SAVE_TASK_FAILED, SAVE_TASK_FAILED), (SAVE_TASK_RUNNING, SAVE_TASK_RUNNING) ] class SaveOriginRequest(models.Model): """ Model table holding all the save origin requests issued by users. """ id = models.BigAutoField(primary_key=True) request_date = models.DateTimeField(auto_now_add=True) - origin_type = models.CharField(max_length=200, null=False) + visit_type = models.CharField(max_length=200, null=False) origin_url = models.CharField(max_length=200, null=False) status = models.TextField(choices=SAVE_REQUEST_STATUS, default=SAVE_REQUEST_PENDING) loading_task_id = models.IntegerField(default=-1) visit_date = models.DateTimeField(null=True) loading_task_status = models.TextField(choices=SAVE_TASK_STATUS, default=SAVE_TASK_NOT_CREATED) class Meta: app_label = 'swh.web.common' db_table = 'save_origin_request' ordering = ['-id'] def __str__(self): return str({'id': self.id, 'request_date': self.request_date, - 'origin_type': self.origin_type, + 'visit_type': self.visit_type, 'origin_url': self.origin_url, 'status': self.status, 'loading_task_id': self.loading_task_id, 'visit_date': self.visit_date}) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py index e21e13f3..28eaee57 100644 --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -1,532 +1,532 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import logging from bisect import bisect_right from datetime import datetime, timezone, timedelta import requests from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError from django.core.validators import URLValidator from django.utils.html import escape from swh.web import config from swh.web.common import service from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc from swh.web.common.models import ( SaveUnauthorizedOrigin, SaveAuthorizedOrigin, SaveOriginRequest, SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_REQUEST_PENDING, SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_SCHEDULED, SAVE_TASK_SUCCEED, SAVE_TASK_FAILED, SAVE_TASK_RUNNING ) from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import parse_timestamp from swh.scheduler.utils import create_oneshot_task_dict scheduler = config.scheduler() logger = logging.getLogger(__name__) def get_origin_save_authorized_urls(): """ Get the list of origin url prefixes authorized to be immediately loaded into the archive (whitelist). Returns: list: The list of authorized origin url prefix """ return [origin.url for origin in SaveAuthorizedOrigin.objects.all()] def get_origin_save_unauthorized_urls(): """ Get the list of origin url prefixes forbidden to be loaded into the archive (blacklist). Returns: list: the list of unauthorized origin url prefix """ return [origin.url for origin in SaveUnauthorizedOrigin.objects.all()] def can_save_origin(origin_url): """ Check if a software origin can be saved into the archive. Based on the origin url, the save request will be either: * immediately accepted if the url is whitelisted * rejected if the url is blacklisted * put in pending state for manual review otherwise Args: origin_url (str): the software origin url to check Returns: str: the origin save request status, either **accepted**, **rejected** or **pending** """ # origin url may be blacklisted for url_prefix in get_origin_save_unauthorized_urls(): if origin_url.startswith(url_prefix): return SAVE_REQUEST_REJECTED # if the origin url is in the white list, it can be immediately saved for url_prefix in get_origin_save_authorized_urls(): if origin_url.startswith(url_prefix): return SAVE_REQUEST_ACCEPTED # otherwise, the origin url needs to be manually verified return SAVE_REQUEST_PENDING -# map origin type to scheduler task +# map visit type to scheduler task # TODO: do not hardcode the task name here (T1157) -_origin_type_task = { +_visit_type_task = { 'git': 'load-git', 'hg': 'load-hg', 'svn': 'load-svn' } # map scheduler task status to origin save status _save_task_status = { 'next_run_not_scheduled': SAVE_TASK_NOT_YET_SCHEDULED, 'next_run_scheduled': SAVE_TASK_SCHEDULED, 'completed': SAVE_TASK_SUCCEED, 'disabled': SAVE_TASK_FAILED } -def get_savable_origin_types(): - return sorted(list(_origin_type_task.keys())) +def get_savable_visit_types(): + return sorted(list(_visit_type_task.keys())) -def _check_origin_type_savable(origin_type): +def _check_visit_type_savable(visit_type): """ - Get the list of software origin types that can be loaded + Get the list of visit types that can be performed through a save request. Returns: - list: the list of saveable origin types + list: the list of saveable visit types """ - allowed_origin_types = ', '.join(get_savable_origin_types()) - if origin_type not in _origin_type_task: - raise BadInputExc('Origin of type %s can not be saved! ' + allowed_visit_types = ', '.join(get_savable_visit_types()) + if visit_type not in _visit_type_task: + raise BadInputExc('Visit of type %s can not be saved! ' 'Allowed types are the following: %s' % - (origin_type, allowed_origin_types)) + (visit_type, allowed_visit_types)) _validate_url = URLValidator(schemes=['http', 'https', 'svn', 'git']) def _check_origin_url_valid(origin_url): try: _validate_url(origin_url) except ValidationError: raise BadInputExc('The provided origin url (%s) is not valid!' % escape(origin_url)) def _get_visit_info_for_save_request(save_request): visit_date = None visit_status = None try: - origin = {'type': save_request.origin_type, - 'url': save_request.origin_url} + origin = {'url': save_request.origin_url} origin_info = service.lookup_origin(origin) origin_visits = get_origin_visits(origin_info) visit_dates = [parse_timestamp(v['date']) for v in origin_visits] i = bisect_right(visit_dates, save_request.request_date) if i != len(visit_dates): visit_date = visit_dates[i] visit_status = origin_visits[i]['status'] if origin_visits[i]['status'] == 'ongoing': visit_date = None except Exception: pass return visit_date, visit_status def _check_visit_update_status(save_request, save_task_status): visit_date, visit_status = _get_visit_info_for_save_request(save_request) save_request.visit_date = visit_date # visit has been performed, mark the saving task as succeed if visit_date and visit_status is not None: save_task_status = SAVE_TASK_SUCCEED elif visit_status == 'ongoing': save_task_status = SAVE_TASK_RUNNING else: time_now = datetime.now(tz=timezone.utc) time_delta = time_now - save_request.request_date # consider the task as failed if it is still in scheduled state # 30 days after its submission if time_delta.days > 30: save_task_status = SAVE_TASK_FAILED return visit_date, save_task_status def _save_request_dict(save_request, task=None): must_save = False visit_date = save_request.visit_date # save task still in scheduler db if task: save_task_status = _save_task_status[task['status']] # Consider request from which a visit date has already been found # as succeeded to avoid retrieving it again if save_task_status == SAVE_TASK_SCHEDULED and visit_date: save_task_status = SAVE_TASK_SUCCEED if save_task_status in (SAVE_TASK_FAILED, SAVE_TASK_SUCCEED) \ and not visit_date: visit_date, _ = _get_visit_info_for_save_request(save_request) save_request.visit_date = visit_date must_save = True # Check tasks still marked as scheduled / not yet scheduled if save_task_status in (SAVE_TASK_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED): visit_date, save_task_status = _check_visit_update_status( save_request, save_task_status) # save task may have been archived else: save_task_status = save_request.loading_task_status if save_task_status in (SAVE_TASK_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED): visit_date, save_task_status = _check_visit_update_status( save_request, save_task_status) else: save_task_status = save_request.loading_task_status if save_request.loading_task_status != save_task_status: save_request.loading_task_status = save_task_status must_save = True if must_save: save_request.save() return {'id': save_request.id, - 'origin_type': save_request.origin_type, + 'visit_type': save_request.visit_type, 'origin_url': save_request.origin_url, 'save_request_date': save_request.request_date.isoformat(), 'save_request_status': save_request.status, 'save_task_status': save_task_status, 'visit_date': visit_date.isoformat() if visit_date else None} -def create_save_origin_request(origin_type, origin_url): +def create_save_origin_request(visit_type, origin_url): """ Create a loading task to save a software origin into the archive. This function aims to create a software origin loading task trough the use of the swh-scheduler component. - First, some checks are performed to see if the origin type and + First, some checks are performed to see if the visit type and origin url are valid but also if the the save request can be accepted. If those checks passed, the loading task is then created. Otherwise, the save request is put in pending or rejected state. All the submitted save requests are logged into the swh-web database to keep track of them. Args: - origin_type (str): the type of origin to save (currently only + visit_type (str): the type of visit to perform (currently only ``git`` but ``svn`` and ``hg`` will soon be available) origin_url (str): the url of the origin to save Raises: - BadInputExc: the origin type or url is invalid + BadInputExc: the visit type or origin url is invalid ForbiddenExc: the provided origin url is blacklisted Returns: dict: A dict describing the save request with the following keys: - * **origin_type**: the type of the origin to save + * **visit_type**: the type of visit to perform * **origin_url**: the url of the origin * **save_request_date**: the date the request was submitted * **save_request_status**: the request status, either **accepted**, **rejected** or **pending** * **save_task_status**: the origin loading task status, either **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** """ - _check_origin_type_savable(origin_type) + _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) save_request_status = can_save_origin(origin_url) task = None # if the origin save request is accepted, create a scheduler # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority kwargs = {'priority': 'high'} - # set task parameters according to the origin type - if origin_type == 'git': + # set task parameters according to the visit type + if visit_type == 'git': kwargs['repo_url'] = origin_url - elif origin_type == 'hg': + elif visit_type == 'hg': kwargs['origin_url'] = origin_url - elif origin_type == 'svn': + elif visit_type == 'svn': kwargs['origin_url'] = origin_url kwargs['svn_url'] = origin_url sor = None # get list of previously sumitted save requests current_sors = \ - list(SaveOriginRequest.objects.filter(origin_type=origin_type, + list(SaveOriginRequest.objects.filter(visit_type=visit_type, origin_url=origin_url)) can_create_task = False # if no save requests previously submitted, create the scheduler task if not current_sors: can_create_task = True else: # get the latest submitted save request sor = current_sors[0] # if it was in pending state, we need to create the scheduler task # and update the save request info in the database if sor.status == SAVE_REQUEST_PENDING: can_create_task = True # a task has already been created to load the origin elif sor.loading_task_id != -1: # get the scheduler task and its status tasks = scheduler.get_tasks([sor.loading_task_id]) task = tasks[0] if tasks else None task_status = _save_request_dict(sor, task)['save_task_status'] # create a new scheduler task only if the previous one has been # already executed if task_status == SAVE_TASK_FAILED or \ task_status == SAVE_TASK_SUCCEED: can_create_task = True sor = None else: can_create_task = False if can_create_task: # effectively create the scheduler task task_dict = create_oneshot_task_dict( - _origin_type_task[origin_type], **kwargs) + _visit_type_task[visit_type], **kwargs) task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted if sor: sor.status = SAVE_REQUEST_ACCEPTED sor.loading_task_id = task['id'] sor.save() else: - sor = SaveOriginRequest.objects.create(origin_type=origin_type, + sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status, # noqa loading_task_id=task['id']) # noqa # save request must be manually reviewed for acceptation elif save_request_status == SAVE_REQUEST_PENDING: # check if there is already such a save request already submitted, # no need to add it to the database in that case try: - sor = SaveOriginRequest.objects.get(origin_type=origin_type, + sor = SaveOriginRequest.objects.get(visit_type=visit_type, origin_url=origin_url, status=save_request_status) # if not add it to the database except ObjectDoesNotExist: - sor = SaveOriginRequest.objects.create(origin_type=origin_type, + sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status) # origin can not be saved as its url is blacklisted, # log the request to the database anyway else: - sor = SaveOriginRequest.objects.create(origin_type=origin_type, + sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status) if save_request_status == SAVE_REQUEST_REJECTED: raise ForbiddenExc('The origin url is blacklisted and will not be ' 'loaded into the archive.') return _save_request_dict(sor, task) def get_save_origin_requests_from_queryset(requests_queryset): """ Get all save requests from a SaveOriginRequest queryset. Args: requests_queryset (django.db.models.QuerySet): input SaveOriginRequest queryset Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ task_ids = [] for sor in requests_queryset: task_ids.append(sor.loading_task_id) save_requests = [] if task_ids: tasks = scheduler.get_tasks(task_ids) tasks = {task['id']: task for task in tasks} for sor in requests_queryset: sr_dict = _save_request_dict(sor, tasks.get(sor.loading_task_id)) save_requests.append(sr_dict) return save_requests -def get_save_origin_requests(origin_type, origin_url): +def get_save_origin_requests(visit_type, origin_url): """ Get all save requests for a given software origin. Args: - origin_type (str): the type of the origin + visit_type (str): the type of visit origin_url (str): the url of the origin Raises: - BadInputExc: the origin type or url is invalid + BadInputExc: the visit type or origin url is invalid NotFoundExc: no save requests can be found for the given origin Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ - _check_origin_type_savable(origin_type) + _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) - sors = SaveOriginRequest.objects.filter(origin_type=origin_type, + sors = SaveOriginRequest.objects.filter(visit_type=visit_type, origin_url=origin_url) if sors.count() == 0: - raise NotFoundExc(('No save requests found for origin with type ' - '%s and url %s.') % (origin_type, origin_url)) + raise NotFoundExc(('No save requests found for visit of type ' + '%s on origin with url %s.') + % (visit_type, origin_url)) return get_save_origin_requests_from_queryset(sors) def get_save_origin_task_info(save_request_id): """ Get detailed information about an accepted save origin request and its associated loading task. If the associated loading task info is archived and removed from the scheduler database, returns an empty dictionary. Args: save_request_id (int): identifier of a save origin request Returns: dict: A dictionary with the following keys: - **type**: loading task type - **arguments**: loading task arguments - **id**: loading task database identifier - **backend_id**: loading task celery identifier - **scheduled**: loading task scheduling date - **ended**: loading task termination date - **status**: loading task execution status Depending on the availability of the task logs in the elasticsearch cluster of Software Heritage, the returned dictionary may also contain the following keys: - **name**: associated celery task name - **message**: relevant log message from task execution - **duration**: task execution time (only if it succeeded) - **worker**: name of the worker that executed the task """ try: save_request = SaveOriginRequest.objects.get(id=save_request_id) except ObjectDoesNotExist: return {} task = scheduler.get_tasks([save_request.loading_task_id]) task = task[0] if task else None if task is None: return {} task_run = scheduler.get_task_runs([task['id']]) task_run = task_run[0] if task_run else None if task_run is None: return {} task_run['type'] = task['type'] task_run['arguments'] = task['arguments'] task_run['id'] = task_run['task'] del task_run['task'] del task_run['metadata'] del task_run['started'] es_workers_index_url = config.get_config()['es_workers_index_url'] if not es_workers_index_url: return task_run es_workers_index_url += '/_search' if save_request.visit_date: min_ts = save_request.visit_date max_ts = min_ts + timedelta(days=7) else: min_ts = save_request.request_date max_ts = min_ts + timedelta(days=30) min_ts = int(min_ts.timestamp()) * 1000 max_ts = int(max_ts.timestamp()) * 1000 save_task_status = _save_task_status[task['status']] priority = '3' if save_task_status == SAVE_TASK_FAILED else '6' query = { 'bool': { 'must': [ { 'match_phrase': { 'priority': { 'query': priority } } }, { 'match_phrase': { 'swh_task_id': { 'query': task_run['backend_id'] } } }, { 'range': { '@timestamp': { 'gte': min_ts, 'lte': max_ts, 'format': 'epoch_millis' } } } ] } } try: response = requests.post(es_workers_index_url, json={'query': query, 'sort': ['@timestamp']}, timeout=30) results = json.loads(response.text) if results['hits']['total'] >= 1: task_run_info = results['hits']['hits'][-1]['_source'] if 'swh_logging_args_runtime' in task_run_info: duration = task_run_info['swh_logging_args_runtime'] task_run['duration'] = duration if 'message' in task_run_info: task_run['message'] = task_run_info['message'] if 'swh_logging_args_name' in task_run_info: task_run['name'] = task_run_info['swh_logging_args_name'] elif 'swh_task_name' in task_run_info: task_run['name'] = task_run_info['swh_task_name'] if 'hostname' in task_run_info: task_run['worker'] = task_run_info['hostname'] elif 'host' in task_run_info: task_run['worker'] = task_run_info['host'] except Exception as e: logger.warning('Request to Elasticsearch failed\n%s' % str(e)) pass return task_run diff --git a/swh/web/common/service.py b/swh/web/common/service.py index eca4f632..b9daca38 100644 --- a/swh/web/common/service.py +++ b/swh/web/common/service.py @@ -1,1103 +1,1101 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from collections import defaultdict from swh.model import hashutil from swh.storage.algos import diff, revisions_walker from swh.web.common import converters from swh.web.common import query from swh.web.common.exc import NotFoundExc from swh.web.common.origin_visits import get_origin_visit from swh.web import config storage = config.storage() vault = config.vault() idx_storage = config.indexer_storage() MAX_LIMIT = 50 # Top limit the users can ask for def _first_element(l): """Returns the first element in the provided list or None if it is empty or None""" return next(iter(l or []), None) def lookup_multiple_hashes(hashes): """Lookup the passed hashes in a single DB connection, using batch processing. Args: An array of {filename: X, sha1: Y}, string X, hex sha1 string Y. Returns: The same array with elements updated with elem['found'] = true if the hash is present in storage, elem['found'] = false if not. """ hashlist = [hashutil.hash_to_bytes(elem['sha1']) for elem in hashes] content_missing = storage.content_missing_per_sha1(hashlist) missing = [hashutil.hash_to_hex(x) for x in content_missing] for x in hashes: x.update({'found': True}) for h in hashes: if h['sha1'] in missing: h['found'] = False return hashes def lookup_expression(expression, last_sha1, per_page): """Lookup expression in raw content. Args: expression (str): An expression to lookup through raw indexed content last_sha1 (str): Last sha1 seen per_page (int): Number of results per page Yields: ctags whose content match the expression """ limit = min(per_page, MAX_LIMIT) ctags = idx_storage.content_ctags_search(expression, last_sha1=last_sha1, limit=limit) for ctag in ctags: ctag = converters.from_swh(ctag, hashess={'id'}) ctag['sha1'] = ctag['id'] ctag.pop('id') yield ctag def lookup_hash(q): """Checks if the storage contains a given content checksum Args: query string of the form Returns: Dict with key found containing the hash info if the hash is present, None if not. """ algo, hash = query.parse_hash(q) found = _first_element(storage.content_find({algo: hash})) return {'found': converters.from_content(found), 'algo': algo} def search_hash(q): """Checks if the storage contains a given content checksum Args: query string of the form Returns: Dict with key found to True or False, according to whether the checksum is present or not """ algo, hash = query.parse_hash(q) found = _first_element(storage.content_find({algo: hash})) return {'found': found is not None} def _lookup_content_sha1(q): """Given a possible input, query for the content's sha1. Args: q: query string of the form Returns: binary sha1 if found or None """ algo, hash = query.parse_hash(q) if algo != 'sha1': hashes = _first_element(storage.content_find({algo: hash})) if not hashes: return None return hashes['sha1'] return hash def lookup_content_ctags(q): """Return ctags information from a specified content. Args: q: query string of the form Yields: ctags information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None ctags = list(idx_storage.content_ctags_get([sha1])) if not ctags: return None for ctag in ctags: yield converters.from_swh(ctag, hashess={'id'}) def lookup_content_filetype(q): """Return filetype information from a specified content. Args: q: query string of the form Yields: filetype information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None filetype = _first_element(list(idx_storage.content_mimetype_get([sha1]))) if not filetype: return None return converters.from_filetype(filetype) def lookup_content_language(q): """Return language information from a specified content. Args: q: query string of the form Yields: language information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None lang = _first_element(list(idx_storage.content_language_get([sha1]))) if not lang: return None return converters.from_swh(lang, hashess={'id'}) def lookup_content_license(q): """Return license information from a specified content. Args: q: query string of the form Yields: license information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None lic = _first_element(idx_storage.content_fossology_license_get([sha1])) if not lic: return None return converters.from_swh({'id': sha1, 'facts': lic[sha1]}, hashess={'id'}) def lookup_origin(origin): """Return information about the origin matching dict origin. Args: - origin: origin's dict with keys either 'id' or 'url' + origin: origin's dict with 'url' key Returns: origin information as dict. """ origin_info = storage.origin_get(origin) if not origin_info: - msg = 'Origin %s not found!' % \ - (origin.get('id') or origin['url']) + msg = 'Origin with url %s not found!' % origin['url'] raise NotFoundExc(msg) return converters.from_origin(origin_info) def lookup_origins(origin_from=1, origin_count=100): """Get list of archived software origins in a paginated way. Origins are sorted by id before returning them Args: origin_from (int): The minimum id of the origins to return origin_count (int): The maximum number of origins to return Yields: origins information as dicts """ origins = storage.origin_get_range(origin_from, origin_count) return map(converters.from_origin, origins) def search_origin(url_pattern, offset=0, limit=50, regexp=False, with_visit=False): """Search for origins whose urls contain a provided string pattern or match a provided regular expression. Args: url_pattern: the string pattern to search for in origin urls offset: number of found origins to skip before returning results limit: the maximum number of found origins to return Returns: list of origin information as dict. """ origins = storage.origin_search(url_pattern, offset, limit, regexp, with_visit) return map(converters.from_origin, origins) def search_origin_metadata(fulltext, limit=50): """Search for origins whose metadata match a provided string pattern. Args: fulltext: the string pattern to search for in origin metadata offset: number of found origins to skip before returning results limit: the maximum number of found origins to return Returns: list of origin metadata as dict. """ matches = idx_storage.origin_intrinsic_metadata_search_fulltext( conjunction=[fulltext], limit=limit) results = [] for match in matches: match['from_revision'] = hashutil.hash_to_hex(match['from_revision']) if match['origin_url']: origin = storage.origin_get({'url': match['origin_url']}) else: # Fallback to origin-id for idx-storage with outdated db origin = storage.origin_get({'id': match['id']}) del match['origin_url'] if 'id' in match: del match['id'] result = converters.from_origin(origin) if result: result['metadata'] = match results.append(result) return results def lookup_origin_intrinsic_metadata(origin_dict): """Return intrinsic metadata for origin whose origin matches given origin. Args: origin_dict: origin's dict with keys ('type' AND 'url') Returns: origin metadata. """ origin_info = storage.origin_get(origin_dict) if not origin_info: - msg = 'Origin with type %s and url %s not found!' % \ - (origin_dict['type'], origin_dict['url']) + msg = 'Origin with url %s not found!' % origin_dict['url'] raise NotFoundExc(msg) origins = [origin_info['url']] match = _first_element( idx_storage.origin_intrinsic_metadata_get(origins)) result = {} if match: result = match['metadata'] return result def _to_sha1_bin(sha1_hex): _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( sha1_hex, ['sha1'], # HACK: sha1_git really 'Only sha1_git is supported.') return sha1_git_bin def _check_directory_exists(sha1_git, sha1_git_bin): if len(list(storage.directory_missing([sha1_git_bin]))): raise NotFoundExc('Directory with sha1_git %s not found' % sha1_git) def lookup_directory(sha1_git): """Return information about the directory with id sha1_git. Args: sha1_git as string Returns: directory information as dict. """ empty_dir_sha1 = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' if sha1_git == empty_dir_sha1: return [] sha1_git_bin = _to_sha1_bin(sha1_git) _check_directory_exists(sha1_git, sha1_git_bin) directory_entries = storage.directory_ls(sha1_git_bin) return map(converters.from_directory_entry, directory_entries) def lookup_directory_with_path(sha1_git, path_string): """Return directory information for entry with path path_string w.r.t. root directory pointed by directory_sha1_git Args: - directory_sha1_git: sha1_git corresponding to the directory to which we append paths to (hopefully) find the entry - the relative path to the entry starting from the directory pointed by directory_sha1_git Raises: NotFoundExc if the directory entry is not found """ sha1_git_bin = _to_sha1_bin(sha1_git) _check_directory_exists(sha1_git, sha1_git_bin) paths = path_string.strip(os.path.sep).split(os.path.sep) queried_dir = storage.directory_entry_get_by_path( sha1_git_bin, list(map(lambda p: p.encode('utf-8'), paths))) if not queried_dir: raise NotFoundExc(('Directory entry with path %s from %s not found') % (path_string, sha1_git)) return converters.from_directory_entry(queried_dir) def lookup_release(release_sha1_git): """Return information about the release with sha1 release_sha1_git. Args: release_sha1_git: The release's sha1 as hexadecimal Returns: Release information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ sha1_git_bin = _to_sha1_bin(release_sha1_git) release = _first_element(storage.release_get([sha1_git_bin])) if not release: raise NotFoundExc('Release with sha1_git %s not found.' % release_sha1_git) return converters.from_release(release) def lookup_release_multiple(sha1_git_list): """Return information about the revisions identified with their sha1_git identifiers. Args: sha1_git_list: A list of revision sha1_git identifiers Returns: Release information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ sha1_bin_list = (_to_sha1_bin(sha1_git) for sha1_git in sha1_git_list) releases = storage.release_get(sha1_bin_list) or [] return (converters.from_release(r) for r in releases) def lookup_revision(rev_sha1_git): """Return information about the revision with sha1 revision_sha1_git. Args: revision_sha1_git: The revision's sha1 as hexadecimal Returns: Revision information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. NotFoundExc if there is no revision with the provided sha1_git. """ sha1_git_bin = _to_sha1_bin(rev_sha1_git) revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: raise NotFoundExc('Revision with sha1_git %s not found.' % rev_sha1_git) return converters.from_revision(revision) def lookup_revision_multiple(sha1_git_list): """Return information about the revisions identified with their sha1_git identifiers. Args: sha1_git_list: A list of revision sha1_git identifiers Returns: Generator of revisions information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ sha1_bin_list = (_to_sha1_bin(sha1_git) for sha1_git in sha1_git_list) revisions = storage.revision_get(sha1_bin_list) or [] return (converters.from_revision(r) for r in revisions) def lookup_revision_message(rev_sha1_git): """Return the raw message of the revision with sha1 revision_sha1_git. Args: revision_sha1_git: The revision's sha1 as hexadecimal Returns: Decoded revision message as dict {'message': } Raises: ValueError if the identifier provided is not of sha1 nature. NotFoundExc if the revision is not found, or if it has no message """ sha1_git_bin = _to_sha1_bin(rev_sha1_git) revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: raise NotFoundExc('Revision with sha1_git %s not found.' % rev_sha1_git) if 'message' not in revision: raise NotFoundExc('No message for revision with sha1_git %s.' % rev_sha1_git) res = {'message': revision['message']} return res def _lookup_revision_id_by(origin, branch_name, timestamp): def _get_snapshot_branch(snapshot, branch_name): snapshot = lookup_snapshot(visit['snapshot'], branches_from=branch_name, branches_count=10) branch = None if branch_name in snapshot['branches']: branch = snapshot['branches'][branch_name] return branch if isinstance(origin, int): origin = {'id': origin} elif isinstance(origin, str): origin = {'url': origin} else: raise TypeError('"origin" must be an int or a string.') visit = get_origin_visit(origin, visit_ts=timestamp) branch = _get_snapshot_branch(visit['snapshot'], branch_name) rev_id = None if branch and branch['target_type'] == 'revision': rev_id = branch['target'] elif branch and branch['target_type'] == 'alias': branch = _get_snapshot_branch(visit['snapshot'], branch['target']) if branch and branch['target_type'] == 'revision': rev_id = branch['target'] if not rev_id: raise NotFoundExc('Revision for origin %s and branch %s not found.' % (origin.get('url'), branch_name)) return rev_id def lookup_revision_by(origin, branch_name='HEAD', timestamp=None): """Lookup revision by origin, snapshot branch name and visit timestamp. If branch_name is not provided, lookup using 'HEAD' as default. If timestamp is not provided, use the most recent. Args: origin (Union[int,str]): origin of the revision branch_name (str): snapshot branch name timestamp (str/int): origin visit time frame Returns: dict: The revision matching the criterions Raises: NotFoundExc if no revision corresponds to the criterion """ rev_id = _lookup_revision_id_by(origin, branch_name, timestamp) return lookup_revision(rev_id) def lookup_revision_log(rev_sha1_git, limit): """Lookup revision log by revision id. Args: rev_sha1_git (str): The revision's sha1 as hexadecimal limit (int): the maximum number of revisions returned Returns: list: Revision log as list of revision dicts Raises: ValueError: if the identifier provided is not of sha1 nature. NotFoundExc: if there is no revision with the provided sha1_git. """ lookup_revision(rev_sha1_git) sha1_git_bin = _to_sha1_bin(rev_sha1_git) revision_entries = storage.revision_log([sha1_git_bin], limit) return map(converters.from_revision, revision_entries) def lookup_revision_log_by(origin, branch_name, timestamp, limit): """Lookup revision by origin, snapshot branch name and visit timestamp. Args: origin (Union[int,str]): origin of the revision branch_name (str): snapshot branch timestamp (str/int): origin visit time frame limit (int): the maximum number of revisions returned Returns: list: Revision log as list of revision dicts Raises: NotFoundExc: if no revision corresponds to the criterion """ rev_id = _lookup_revision_id_by(origin, branch_name, timestamp) return lookup_revision_log(rev_id, limit) def lookup_revision_with_context_by(origin, branch_name, timestamp, sha1_git, limit=100): """Return information about revision sha1_git, limited to the sub-graph of all transitive parents of sha1_git_root. sha1_git_root being resolved through the lookup of a revision by origin, branch_name and ts. In other words, sha1_git is an ancestor of sha1_git_root. Args: - origin: origin of the revision. - branch_name: revision's branch. - timestamp: revision's time frame. - sha1_git: one of sha1_git_root's ancestors. - limit: limit the lookup to 100 revisions back. Returns: Pair of (root_revision, revision). Information on sha1_git if it is an ancestor of sha1_git_root including children leading to sha1_git_root Raises: - BadInputExc in case of unknown algo_hash or bad hash. - NotFoundExc if either revision is not found or if sha1_git is not an ancestor of sha1_git_root. """ rev_root_id = _lookup_revision_id_by(origin, branch_name, timestamp) rev_root_id_bin = hashutil.hash_to_bytes(rev_root_id) rev_root = _first_element(storage.revision_get([rev_root_id_bin])) return (converters.from_revision(rev_root), lookup_revision_with_context(rev_root, sha1_git, limit)) def lookup_revision_with_context(sha1_git_root, sha1_git, limit=100): """Return information about revision sha1_git, limited to the sub-graph of all transitive parents of sha1_git_root. In other words, sha1_git is an ancestor of sha1_git_root. Args: sha1_git_root: latest revision. The type is either a sha1 (as an hex string) or a non converted dict. sha1_git: one of sha1_git_root's ancestors limit: limit the lookup to 100 revisions back Returns: Information on sha1_git if it is an ancestor of sha1_git_root including children leading to sha1_git_root Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if either revision is not found or if sha1_git is not an ancestor of sha1_git_root """ sha1_git_bin = _to_sha1_bin(sha1_git) revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: raise NotFoundExc('Revision %s not found' % sha1_git) if isinstance(sha1_git_root, str): sha1_git_root_bin = _to_sha1_bin(sha1_git_root) revision_root = _first_element(storage.revision_get([sha1_git_root_bin])) # noqa if not revision_root: raise NotFoundExc('Revision root %s not found' % sha1_git_root) else: sha1_git_root_bin = sha1_git_root['id'] revision_log = storage.revision_log([sha1_git_root_bin], limit) parents = {} children = defaultdict(list) for rev in revision_log: rev_id = rev['id'] parents[rev_id] = [] for parent_id in rev['parents']: parents[rev_id].append(parent_id) children[parent_id].append(rev_id) if revision['id'] not in parents: raise NotFoundExc('Revision %s is not an ancestor of %s' % (sha1_git, sha1_git_root)) revision['children'] = children[revision['id']] return converters.from_revision(revision) def lookup_directory_with_revision(sha1_git, dir_path=None, with_data=False): """Return information on directory pointed by revision with sha1_git. If dir_path is not provided, display top level directory. Otherwise, display the directory pointed by dir_path (if it exists). Args: sha1_git: revision's hash. dir_path: optional directory pointed to by that revision. with_data: boolean that indicates to retrieve the raw data if the path resolves to a content. Default to False (for the api) Returns: Information on the directory pointed to by that revision. Raises: BadInputExc in case of unknown algo_hash or bad hash. NotFoundExc either if the revision is not found or the path referenced does not exist. NotImplementedError in case of dir_path exists but do not reference a type 'dir' or 'file'. """ sha1_git_bin = _to_sha1_bin(sha1_git) revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: raise NotFoundExc('Revision %s not found' % sha1_git) dir_sha1_git_bin = revision['directory'] if dir_path: paths = dir_path.strip(os.path.sep).split(os.path.sep) entity = storage.directory_entry_get_by_path( dir_sha1_git_bin, list(map(lambda p: p.encode('utf-8'), paths))) if not entity: raise NotFoundExc( "Directory or File '%s' pointed to by revision %s not found" % (dir_path, sha1_git)) else: entity = {'type': 'dir', 'target': dir_sha1_git_bin} if entity['type'] == 'dir': directory_entries = storage.directory_ls(entity['target']) or [] return {'type': 'dir', 'path': '.' if not dir_path else dir_path, 'revision': sha1_git, 'content': list(map(converters.from_directory_entry, directory_entries))} elif entity['type'] == 'file': # content content = _first_element( storage.content_find({'sha1_git': entity['target']})) if not content: raise NotFoundExc('Content not found for revision %s' % sha1_git) if with_data: c = _first_element(storage.content_get([content['sha1']])) content['data'] = c['data'] return {'type': 'file', 'path': '.' if not dir_path else dir_path, 'revision': sha1_git, 'content': converters.from_content(content)} elif entity['type'] == 'rev': # revision revision = next(storage.revision_get([entity['target']])) return {'type': 'rev', 'path': '.' if not dir_path else dir_path, 'revision': sha1_git, 'content': converters.from_revision(revision)} else: raise NotImplementedError('Entity of type %s not implemented.' % entity['type']) def lookup_content(q): """Lookup the content designed by q. Args: q: The release's sha1 as hexadecimal Raises: NotFoundExc if the requested content is not found """ algo, hash = query.parse_hash(q) c = _first_element(storage.content_find({algo: hash})) if not c: raise NotFoundExc('Content with %s checksum equals to %s not found!' % (algo, hashutil.hash_to_hex(hash))) return converters.from_content(c) def lookup_content_raw(q): """Lookup the content defined by q. Args: q: query string of the form Returns: dict with 'sha1' and 'data' keys. data representing its raw data decoded. Raises: NotFoundExc if the requested content is not found or if the content bytes are not available in the storage """ c = lookup_content(q) content_sha1_bytes = hashutil.hash_to_bytes(c['checksums']['sha1']) content = _first_element(storage.content_get([content_sha1_bytes])) if not content: algo, hash = query.parse_hash(q) raise NotFoundExc('Bytes of content with %s checksum equals to %s ' 'are not available!' % (algo, hashutil.hash_to_hex(hash))) return converters.from_content(content) def stat_counters(): """Return the stat counters for Software Heritage Returns: A dict mapping textual labels to integer values. """ return storage.stat_counters() def _lookup_origin_visits(origin_url, last_visit=None, limit=10): """Yields the origin origins' visits. Args: origin_url (str): origin to list visits for last_visit (int): last visit to lookup from limit (int): Number of elements max to display Yields: Dictionaries of origin_visit for that origin """ limit = min(limit, MAX_LIMIT) for visit in storage.origin_visit_get( origin_url, last_visit=last_visit, limit=limit): visit['origin'] = origin_url yield visit def lookup_origin_visits(origin, last_visit=None, per_page=10): """Yields the origin origins' visits. Args: origin: origin to list visits for Yields: Dictionaries of origin_visit for that origin """ visits = _lookup_origin_visits(origin, last_visit=last_visit, limit=per_page) for visit in visits: yield converters.from_origin_visit(visit) def lookup_origin_visit_latest(origin_url, require_snapshot): """Return the origin's latest visit Args: origin_url (str): origin to list visits for require_snapshot (bool): filter out origins without a snapshot Returns: dict: The origin_visit concerned """ visit = storage.origin_visit_get_latest( origin_url, require_snapshot=require_snapshot) if isinstance(visit['origin'], int): # soon-to-be-legacy origin ids visit['origin'] = storage.origin_get({'id': visit['origin']})['url'] return converters.from_origin_visit(visit) def lookup_origin_visit(origin_url, visit_id): """Return information about visit visit_id with origin origin. Args: origin (str): origin concerned by the visit visit_id: the visit identifier to lookup Yields: The dict origin_visit concerned """ visit = storage.origin_visit_get_by(origin_url, visit_id) if not visit: raise NotFoundExc('Origin %s or its visit ' 'with id %s not found!' % (origin_url, visit_id)) visit['origin'] = origin_url return converters.from_origin_visit(visit) def lookup_snapshot_size(snapshot_id): """Count the number of branches in the snapshot with the given id Args: snapshot_id (str): sha1 identifier of the snapshot Returns: dict: A dict whose keys are the target types of branches and values their corresponding amount """ snapshot_id_bin = _to_sha1_bin(snapshot_id) snapshot_size = storage.snapshot_count_branches(snapshot_id_bin) if 'revision' not in snapshot_size: snapshot_size['revision'] = 0 if 'release' not in snapshot_size: snapshot_size['release'] = 0 return snapshot_size def lookup_snapshot(snapshot_id, branches_from='', branches_count=1000, target_types=None): """Return information about a snapshot, aka the list of named branches found during a specific visit of an origin. Args: snapshot_id (str): sha1 identifier of the snapshot branches_from (str): optional parameter used to skip branches whose name is lesser than it before returning them branches_count (int): optional parameter used to restrain the amount of returned branches target_types (list): optional parameter used to filter the target types of branch to return (possible values that can be contained in that list are `'content', 'directory', 'revision', 'release', 'snapshot', 'alias'`) Returns: A dict filled with the snapshot content. """ snapshot_id_bin = _to_sha1_bin(snapshot_id) snapshot = storage.snapshot_get_branches(snapshot_id_bin, branches_from.encode(), branches_count, target_types) if not snapshot: raise NotFoundExc('Snapshot with id %s not found!' % snapshot_id) return converters.from_snapshot(snapshot) def lookup_latest_origin_snapshot(origin, allowed_statuses=None): """Return information about the latest snapshot of an origin. .. warning:: At most 1000 branches contained in the snapshot will be returned for performance reasons. Args: origin: URL or integer identifier of the origin allowed_statuses: list of visit statuses considered to find the latest snapshot for the visit. For instance, ``allowed_statuses=['full']`` will only consider visits that have successfully run to completion. Returns: A dict filled with the snapshot content. """ snapshot = storage.snapshot_get_latest(origin, allowed_statuses) return converters.from_snapshot(snapshot) def lookup_revision_through(revision, limit=100): """Retrieve a revision from the criterion stored in revision dictionary. Args: revision: Dictionary of criterion to lookup the revision with. Here are the supported combination of possible values: - origin_url, branch_name, ts, sha1_git - origin_url, branch_name, ts - sha1_git_root, sha1_git - sha1_git Returns: None if the revision is not found or the actual revision. """ if ( 'origin_url' in revision and 'branch_name' in revision and 'ts' in revision and 'sha1_git' in revision): return lookup_revision_with_context_by(revision['origin_url'], revision['branch_name'], revision['ts'], revision['sha1_git'], limit) if ( 'origin_url' in revision and 'branch_name' in revision and 'ts' in revision): return lookup_revision_by(revision['origin_url'], revision['branch_name'], revision['ts']) if ( 'sha1_git_root' in revision and 'sha1_git' in revision): return lookup_revision_with_context(revision['sha1_git_root'], revision['sha1_git'], limit) if 'sha1_git' in revision: return lookup_revision(revision['sha1_git']) # this should not happen raise NotImplementedError('Should not happen!') def lookup_directory_through_revision(revision, path=None, limit=100, with_data=False): """Retrieve the directory information from the revision. Args: revision: dictionary of criterion representing a revision to lookup path: directory's path to lookup. limit: optional query parameter to limit the revisions log (default to 100). For now, note that this limit could impede the transitivity conclusion about sha1_git not being an ancestor of. with_data: indicate to retrieve the content's raw data if path resolves to a content. Returns: The directory pointing to by the revision criterions at path. """ rev = lookup_revision_through(revision, limit) if not rev: raise NotFoundExc('Revision with criterion %s not found!' % revision) return (rev['id'], lookup_directory_with_revision(rev['id'], path, with_data)) def vault_cook(obj_type, obj_id, email=None): """Cook a vault bundle. """ return vault.cook(obj_type, obj_id, email=email) def vault_fetch(obj_type, obj_id): """Fetch a vault bundle. """ return vault.fetch(obj_type, obj_id) def vault_progress(obj_type, obj_id): """Get the current progress of a vault bundle. """ return vault.progress(obj_type, obj_id) def diff_revision(rev_id): """Get the list of file changes (insertion / deletion / modification / renaming) for a particular revision. """ rev_sha1_git_bin = _to_sha1_bin(rev_id) changes = diff.diff_revision(storage, rev_sha1_git_bin, track_renaming=True) for change in changes: change['from'] = converters.from_directory_entry(change['from']) change['to'] = converters.from_directory_entry(change['to']) if change['from_path']: change['from_path'] = change['from_path'].decode('utf-8') if change['to_path']: change['to_path'] = change['to_path'].decode('utf-8') return changes class _RevisionsWalkerProxy(object): """ Proxy class wrapping a revisions walker iterator from swh-storage and performing needed conversions. """ def __init__(self, rev_walker_type, rev_start, *args, **kwargs): rev_start_bin = hashutil.hash_to_bytes(rev_start) self.revisions_walker = \ revisions_walker.get_revisions_walker(rev_walker_type, storage, rev_start_bin, *args, **kwargs) def export_state(self): return self.revisions_walker.export_state() def __next__(self): return converters.from_revision(next(self.revisions_walker)) def __iter__(self): return self def get_revisions_walker(rev_walker_type, rev_start, *args, **kwargs): """ Utility function to instantiate a revisions walker of a given type, see :mod:`swh.storage.algos.revisions_walker`. Args: rev_walker_type (str): the type of revisions walker to return, possible values are: ``committer_date``, ``dfs``, ``dfs_post``, ``bfs`` and ``path`` rev_start (str): hexadecimal representation of a revision identifier args (list): position arguments to pass to the revisions walker constructor kwargs (dict): keyword arguments to pass to the revisions walker constructor """ # first check if the provided revision is valid lookup_revision(rev_start) return _RevisionsWalkerProxy(rev_walker_type, rev_start, *args, **kwargs) diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py index d7a6c77e..4a3d5949 100644 --- a/swh/web/common/swh_templatetags.py +++ b/swh/web/common/swh_templatetags.py @@ -1,185 +1,185 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import re from django import template from django.core.serializers.json import DjangoJSONEncoder from django.utils.safestring import mark_safe from docutils.core import publish_parts from docutils.writers.html4css1 import Writer, HTMLTranslator from inspect import cleandoc -from swh.web.common.origin_save import get_savable_origin_types +from swh.web.common.origin_save import get_savable_visit_types register = template.Library() class NoHeaderHTMLTranslator(HTMLTranslator): """ Docutils translator subclass to customize the generation of HTML from reST-formatted docstrings """ def __init__(self, document): super().__init__(document) self.body_prefix = [] self.body_suffix = [] def visit_bullet_list(self, node): self.context.append((self.compact_simple, self.compact_p)) self.compact_p = None self.compact_simple = self.is_compactable(node) self.body.append(self.starttag(node, 'ul', CLASS='docstring')) DOCSTRING_WRITER = Writer() DOCSTRING_WRITER.translator_class = NoHeaderHTMLTranslator @register.filter def safe_docstring_display(docstring): """ Utility function to htmlize reST-formatted documentation in browsable api. """ docstring = cleandoc(docstring) return publish_parts(docstring, writer=DOCSTRING_WRITER)['html_body'] @register.filter def urlize_links_and_mails(text): """Utility function for decorating api links in browsable api. Args: text: whose content matching links should be transformed into contextual API or Browse html links. Returns The text transformed if any link is found. The text as is otherwise. """ try: if 'href="' not in text: text = re.sub(r'(/api/[^"<]*|/browse/[^"<]*|http.*$)', r'\1', text) return re.sub(r'([^ <>"]+@[^ <>"]+)', r'\1', text) except Exception: pass return text @register.filter def urlize_header_links(text): """Utility function for decorating headers links in browsable api. Args text: Text whose content contains Link header value Returns: The text transformed with html link if any link is found. The text as is otherwise. """ links = text.split(',') ret = '' for i, link in enumerate(links): ret += re.sub(r'<(/api/.*|/browse/.*)>', r'<\1>', link) # add one link per line and align them if i != len(links) - 1: ret += '\n ' return ret @register.filter def jsonify(obj): """Utility function for converting a django template variable to JSON in order to use it in script tags. Args obj: Any django template context variable Returns: JSON representation of the variable. """ return mark_safe(json.dumps(obj, cls=DjangoJSONEncoder)) @register.filter def sub(value, arg): """Django template filter for subtracting two numbers Args: value (int/float): the value to subtract from arg (int/float): the value to subtract to Returns: int/float: The subtraction result """ return value - arg @register.filter def mul(value, arg): """Django template filter for multiplying two numbers Args: value (int/float): the value to multiply from arg (int/float): the value to multiply with Returns: int/float: The multiplication result """ return value * arg @register.filter def key_value(dict, key): """Django template filter to get a value in a dictionary. Args: dict (dict): a dictionary key (str): the key to lookup value Returns: The requested value in the dictionary """ return dict[key] @register.filter -def origin_type_savable(origin_type): +def visit_type_savable(visit_type): """Django template filter to check if a save request can be - created for a given origin type. + created for a given visit type. Args: - origin_type (str): the type of software origin + visit_type (str): the type of visit Returns: - If the origin type is saveable or not + If the visit type is saveable or not """ - return origin_type in get_savable_origin_types() + return visit_type in get_savable_visit_types() @register.filter def split(value, arg): """Django template filter to split a string. Args: value (str): the string to split arg (str): the split separator Returns: list: the split string parts """ return value.split(arg) diff --git a/swh/web/misc/origin_save.py b/swh/web/misc/origin_save.py index 818c2f71..2236ab01 100644 --- a/swh/web/misc/origin_save.py +++ b/swh/web/misc/origin_save.py @@ -1,102 +1,102 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.conf.urls import url from django.core.paginator import Paginator from django.http import HttpResponse, HttpResponseForbidden from django.shortcuts import render from rest_framework.decorators import api_view, authentication_classes from swh.web.common.exc import ForbiddenExc from swh.web.common.models import SaveOriginRequest from swh.web.common.origin_save import ( - create_save_origin_request, get_savable_origin_types, + create_save_origin_request, get_savable_visit_types, get_save_origin_requests_from_queryset ) from swh.web.common.throttling import throttle_scope from swh.web.common.utils import EnforceCSRFAuthentication def _origin_save_view(request): return render(request, 'misc/origin-save.html', {'heading': ('Request the saving of a software origin into ' 'the archive')}) @api_view(['POST']) @authentication_classes((EnforceCSRFAuthentication, )) @throttle_scope('swh_save_origin') -def _origin_save_request(request, origin_type, origin_url): +def _origin_save_request(request, visit_type, origin_url): """ This view is called through AJAX from the save code now form of swh-web. We use DRF here as we want to rate limit the number of submitted requests per user to avoid being possibly flooded by bots. """ try: - response = json.dumps(create_save_origin_request(origin_type, + response = json.dumps(create_save_origin_request(visit_type, origin_url), separators=(',', ': ')) return HttpResponse(response, content_type='application/json') except ForbiddenExc as exc: return HttpResponseForbidden(str(exc)) -def _origin_save_types_list(request): - origin_types = json.dumps(get_savable_origin_types(), - separators=(',', ': ')) - return HttpResponse(origin_types, content_type='application/json') +def _visit_save_types_list(request): + visit_types = json.dumps(get_savable_visit_types(), + separators=(',', ': ')) + return HttpResponse(visit_types, content_type='application/json') def _origin_save_requests_list(request, status): if status != 'all': save_requests = SaveOriginRequest.objects.filter(status=status) else: save_requests = SaveOriginRequest.objects.all() table_data = {} table_data['recordsTotal'] = save_requests.count() table_data['draw'] = int(request.GET['draw']) search_value = request.GET['search[value]'] column_order = request.GET['order[0][column]'] field_order = request.GET['columns[%s][name]' % column_order] order_dir = request.GET['order[0][dir]'] if order_dir == 'desc': field_order = '-' + field_order save_requests = save_requests.order_by(field_order) length = int(request.GET['length']) page = int(request.GET['start']) / length + 1 save_requests = get_save_origin_requests_from_queryset(save_requests) if search_value: save_requests = \ [sr for sr in save_requests if search_value.lower() in sr['save_request_status'].lower() or search_value.lower() in sr['save_task_status'].lower() - or search_value.lower() in sr['origin_type'].lower() + or search_value.lower() in sr['visit_type'].lower() or search_value.lower() in sr['origin_url'].lower()] table_data['recordsFiltered'] = len(save_requests) paginator = Paginator(save_requests, length) table_data['data'] = paginator.page(page).object_list table_data_json = json.dumps(table_data, separators=(',', ': ')) return HttpResponse(table_data_json, content_type='application/json') urlpatterns = [ url(r'^save/$', _origin_save_view, name='origin-save'), - url(r'^save/(?P.+)/url/(?P.+)/$', + url(r'^save/(?P.+)/url/(?P.+)/$', _origin_save_request, name='origin-save-request'), - url(r'^save/types/list/$', _origin_save_types_list, + url(r'^save/types/list/$', _visit_save_types_list, name='origin-save-types-list'), url(r'^save/requests/list/(?P.+)/$', _origin_save_requests_list, name='origin-save-requests-list'), ] diff --git a/swh/web/templates/browse/search.html b/swh/web/templates/browse/search.html index f71616a8..eb2a49c7 100644 --- a/swh/web/templates/browse/search.html +++ b/swh/web/templates/browse/search.html @@ -1,75 +1,75 @@ {% extends "./layout.html" %} {% comment %} Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} {% block navbar-content %}

Search archived software

{% endblock %} {% block browse-content %}

Searching origins ...

{% endblock %} diff --git a/swh/web/templates/includes/take-new-snapshot.html b/swh/web/templates/includes/take-new-snapshot.html index 5b4ffe8a..23ae9a1c 100644 --- a/swh/web/templates/includes/take-new-snapshot.html +++ b/swh/web/templates/includes/take-new-snapshot.html @@ -1,76 +1,76 @@ {% comment %} Copyright (C) 2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} {% load swh_templatetags %} -{% if snapshot_context and snapshot_context.origin_info and snapshot_context.origin_info.type|origin_type_savable %} +{% if snapshot_context and snapshot_context.visit_info and snapshot_context.visit_info.type|visit_type_savable %}
{% endif %} \ No newline at end of file diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html index 21c4ec32..d901972f 100644 --- a/swh/web/templates/misc/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -1,120 +1,120 @@ {% extends "../layout.html" %} {% comment %} Copyright (C) 2018-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load render_bundle from webpack_loader %} {% load static %} {% block title %}{{ heading }} – Software Heritage archive{% endblock %} {% block header %} {% render_bundle 'save' %} {% endblock %} {% block navbar-content %}

Save code now

{% endblock %} {% block content %}

You can contribute to extend the content of the Software Heritage archive by submitting an origin save request. To do so, fill the required info in the form below:

    -
  • Origin type: the type of version control system the software origin is using. +
  • Visit type: the type of version control system the software origin is using. Currently, the supported types are:
  • Origin url: the url of the remote repository for the software origin.
    In order to avoid saving errors from Software Heritage, you should provide the clone/checkout url as given by the provider hosting the software origin.
    It can easily be found in the web interface used to browse the software origin.
    For instance, if you want to save a git origin into the archive, you should check that the command $ git clone <origin_url>
    does not return an error before submitting a request.

Once submitted, your save request can either be:

  • accepted: a visit to the provided origin will then be scheduled by Software Heritage in order to load its content into the archive as soon as possible
  • rejected: the provided origin url is blacklisted and no visit will be scheduled
  • put in pending state: a manual review will then be performed in order to determine if the origin can be safely loaded or not into the archive

Once a save request has been accepted, you can follow its current status in the submitted save requests list.

{% csrf_token %}
- - {% comment %} {% endcomment %} -
The origin type must be specified
+
The visit type must be specified
The origin url is not valid or does not reference a code repository
Date Type Url Request Status

{% endblock %} \ No newline at end of file diff --git a/swh/web/tests/admin/test_origin_save.py b/swh/web/tests/admin/test_origin_save.py index 171cf504..5240f2e4 100644 --- a/swh/web/tests/admin/test_origin_save.py +++ b/swh/web/tests/admin/test_origin_save.py @@ -1,232 +1,232 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from urllib.parse import unquote from django.contrib.auth import get_user_model from unittest.mock import patch from swh.web.common.models import ( SaveAuthorizedOrigin, SaveUnauthorizedOrigin, SaveOriginRequest ) from swh.web.common.origin_save import can_save_origin from swh.web.common.models import ( SAVE_REQUEST_PENDING, SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_TASK_NOT_YET_SCHEDULED ) from swh.web.common.utils import reverse from swh.web.tests.testcase import WebTestCase _user_name = 'swh-web-admin' _user_mail = 'admin@swh-web.org' _user_password = '..34~pounds~BEAUTY~march~63..' _authorized_origin_url = 'https://scm.ourproject.org/anonscm/' _unauthorized_origin_url = 'https://www.softwareheritage.org/' class OriginSaveAdminTestCase(WebTestCase): @classmethod def setUpTestData(cls): # noqa: N802 User = get_user_model() # noqa: N806 user = User.objects.create_user(_user_name, _user_mail, _user_password) user.is_staff = True user.save() SaveAuthorizedOrigin.objects.create(url=_authorized_origin_url) SaveUnauthorizedOrigin.objects.create(url=_unauthorized_origin_url) def check_not_login(self, url): login_url = reverse('login', query_params={'next': url}) response = self.client.post(url) self.assertEqual(response.status_code, 302) self.assertEqual(unquote(response.url), login_url) def test_add_authorized_origin_url(self): authorized_url = 'https://scm.adullact.net/anonscm/' self.assertEqual(can_save_origin(authorized_url), SAVE_REQUEST_PENDING) url = reverse('admin-origin-save-add-authorized-url', url_args={'origin_url': authorized_url}) self.check_not_login(url) self.assertEqual(can_save_origin(authorized_url), SAVE_REQUEST_PENDING) self.client.login(username=_user_name, password=_user_password) response = self.client.post(url) self.assertEqual(response.status_code, 200) self.assertEqual(can_save_origin(authorized_url), SAVE_REQUEST_ACCEPTED) def test_remove_authorized_origin_url(self): self.assertEqual(can_save_origin(_authorized_origin_url), SAVE_REQUEST_ACCEPTED) url = reverse('admin-origin-save-remove-authorized-url', url_args={'origin_url': _authorized_origin_url}) self.check_not_login(url) self.assertEqual(can_save_origin(_authorized_origin_url), SAVE_REQUEST_ACCEPTED) self.client.login(username=_user_name, password=_user_password) response = self.client.post(url) self.assertEqual(response.status_code, 200) self.assertEqual(can_save_origin(_authorized_origin_url), SAVE_REQUEST_PENDING) def test_add_unauthorized_origin_url(self): unauthorized_url = 'https://www.yahoo./' self.assertEqual(can_save_origin(unauthorized_url), SAVE_REQUEST_PENDING) url = reverse('admin-origin-save-add-unauthorized-url', url_args={'origin_url': unauthorized_url}) self.check_not_login(url) self.assertEqual(can_save_origin(unauthorized_url), SAVE_REQUEST_PENDING) self.client.login(username=_user_name, password=_user_password) response = self.client.post(url) self.assertEqual(response.status_code, 200) self.assertEqual(can_save_origin(unauthorized_url), SAVE_REQUEST_REJECTED) def test_remove_unauthorized_origin_url(self): self.assertEqual(can_save_origin(_unauthorized_origin_url), SAVE_REQUEST_REJECTED) url = reverse('admin-origin-save-remove-unauthorized-url', url_args={'origin_url': _unauthorized_origin_url}) self.check_not_login(url) self.assertEqual(can_save_origin(_unauthorized_origin_url), SAVE_REQUEST_REJECTED) self.client.login(username=_user_name, password=_user_password) response = self.client.post(url) self.assertEqual(response.status_code, 200) self.assertEqual(can_save_origin(_unauthorized_origin_url), SAVE_REQUEST_PENDING) @patch('swh.web.common.origin_save.scheduler') def test_accept_pending_save_request(self, mock_scheduler): - origin_type = 'git' + visit_type = 'git' origin_url = 'https://v2.pikacode.com/bthate/botlib.git' save_request_url = reverse('api-1-save-origin', - url_args={'origin_type': origin_type, + url_args={'visit_type': visit_type, 'origin_url': origin_url}) response = self.client.post(save_request_url, data={}, content_type='application/x-www-form-urlencoded') # noqa self.assertEqual(response.status_code, 200) self.assertEqual(response.data['save_request_status'], SAVE_REQUEST_PENDING) accept_request_url = reverse('admin-origin-save-request-accept', - url_args={'origin_type': origin_type, + url_args={'visit_type': visit_type, 'origin_url': origin_url}) self.check_not_login(accept_request_url) tasks_data = [ { 'priority': 'high', 'policy': 'oneshot', 'type': 'load-git', 'arguments': { 'kwargs': { 'repo_url': origin_url }, 'args': [] }, 'status': 'next_run_not_scheduled', 'id': 1, } ] mock_scheduler.create_tasks.return_value = tasks_data mock_scheduler.get_tasks.return_value = tasks_data self.client.login(username=_user_name, password=_user_password) response = self.client.post(accept_request_url) self.assertEqual(response.status_code, 200) response = self.client.get(save_request_url) self.assertEqual(response.status_code, 200) self.assertEqual(response.data[0]['save_request_status'], SAVE_REQUEST_ACCEPTED) self.assertEqual(response.data[0]['save_task_status'], SAVE_TASK_NOT_YET_SCHEDULED) @patch('swh.web.common.origin_save.scheduler') def test_reject_pending_save_request(self, mock_scheduler): - origin_type = 'git' + visit_type = 'git' origin_url = 'https://wikipedia.com' save_request_url = reverse('api-1-save-origin', - url_args={'origin_type': origin_type, + url_args={'visit_type': visit_type, 'origin_url': origin_url}) response = self.client.post(save_request_url, data={}, content_type='application/x-www-form-urlencoded') # noqa self.assertEqual(response.status_code, 200) self.assertEqual(response.data['save_request_status'], SAVE_REQUEST_PENDING) reject_request_url = reverse('admin-origin-save-request-reject', - url_args={'origin_type': origin_type, + url_args={'visit_type': visit_type, 'origin_url': origin_url}) self.check_not_login(reject_request_url) self.client.login(username=_user_name, password=_user_password) response = self.client.post(reject_request_url) self.assertEqual(response.status_code, 200) tasks_data = [ { 'priority': 'high', 'policy': 'oneshot', 'type': 'load-git', 'arguments': { 'kwargs': { 'repo_url': origin_url }, 'args': [] }, 'status': 'next_run_not_scheduled', 'id': 1, } ] mock_scheduler.create_tasks.return_value = tasks_data mock_scheduler.get_tasks.return_value = tasks_data response = self.client.get(save_request_url) self.assertEqual(response.status_code, 200) self.assertEqual(response.data[0]['save_request_status'], SAVE_REQUEST_REJECTED) def test_remove_save_request(self): - sor = SaveOriginRequest.objects.create(origin_type='git', + sor = SaveOriginRequest.objects.create(visit_type='git', origin_url='https://wikipedia.com', # noqa status=SAVE_REQUEST_PENDING) self.assertEqual(SaveOriginRequest.objects.count(), 1) remove_request_url = reverse('admin-origin-save-request-remove', url_args={'sor_id': sor.id}) self.check_not_login(remove_request_url) self.client.login(username=_user_name, password=_user_password) response = self.client.post(remove_request_url) self.assertEqual(response.status_code, 200) self.assertEqual(SaveOriginRequest.objects.count(), 0) diff --git a/swh/web/tests/api/views/test_origin.py b/swh/web/tests/api/views/test_origin.py index 9e59feba..42129b21 100644 --- a/swh/web/tests/api/views/test_origin.py +++ b/swh/web/tests/api/views/test_origin.py @@ -1,643 +1,621 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch from hypothesis import given, strategies import pytest from requests.utils import parse_header_links from rest_framework.test import APITestCase from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.common.utils import reverse from swh.web.common.origin_visits import get_origin_visits from swh.web.tests.strategies import ( origin, new_origin, visit_dates, new_snapshots ) from swh.web.tests.testcase import WebTestCase from swh.web.tests.data import get_tests_data class OriginApiTestCase(WebTestCase, APITestCase): def _scroll_results(self, url): """Iterates through pages of results, and returns them all.""" results = [] while True: rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') results.extend(rv.data) if 'Link' in rv: for link in parse_header_links(rv['Link']): if link['rel'] == 'next': # Found link to next page of results url = link['url'] break else: # No link with 'rel=next' break else: # No Link header break return results @patch('swh.web.api.views.origin.get_origin_visits') def test_api_lookup_origin_visits_raise_error( self, mock_get_origin_visits, ): err_msg = 'voluntary error to check the bad request middleware.' mock_get_origin_visits.side_effect = ValueError(err_msg) url = reverse( 'api-1-origin-visits', url_args={'origin_url': 'http://foo'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 400, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'ValueError', 'reason': err_msg}) @patch('swh.web.api.views.origin.get_origin_visits') def test_api_lookup_origin_visits_raise_swh_storage_error_db( self, mock_get_origin_visits): err_msg = 'Storage exploded! Will be back online shortly!' mock_get_origin_visits.side_effect = StorageDBError(err_msg) url = reverse( 'api-1-origin-visits', url_args={'origin_url': 'http://foo'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 503, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'StorageDBError', 'reason': 'An unexpected error occurred in the backend: %s' % err_msg}) @patch('swh.web.api.views.origin.get_origin_visits') def test_api_lookup_origin_visits_raise_swh_storage_error_api( self, mock_get_origin_visits): err_msg = 'Storage API dropped dead! Will resurrect asap!' mock_get_origin_visits.side_effect = StorageAPIError(err_msg) url = reverse( 'api-1-origin-visits', url_args={'origin_url': 'http://foo'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 503, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'StorageAPIError', 'reason': 'An unexpected error occurred in the api backend: %s' % err_msg }) @given(new_origin(), visit_dates(3), new_snapshots(3)) def test_api_lookup_origin_visits(self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( - new_origin['url'], visit_date) + new_origin['url'], visit_date, type='git') self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) all_visits = list(reversed(get_origin_visits(new_origin))) for last_visit, expected_visits in ( (None, all_visits[:2]), (all_visits[1]['visit'], all_visits[2:4])): url = reverse('api-1-origin-visits', url_args={'origin_url': new_origin['url']}, query_params={'per_page': 2, 'last_visit': last_visit}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') for expected_visit in expected_visits: origin_visit_url = reverse( 'api-1-origin-visit', url_args={'origin_url': new_origin['url'], 'visit_id': expected_visit['visit']}) snapshot_url = reverse( 'api-1-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_visit_url'] = origin_visit_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visits) @given(new_origin(), visit_dates(3), new_snapshots(3)) def test_api_lookup_origin_visits_by_id(self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( - new_origin['url'], visit_date) + new_origin['url'], visit_date, type='git') self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) all_visits = list(reversed(get_origin_visits(new_origin))) for last_visit, expected_visits in ( (None, all_visits[:2]), (all_visits[1]['visit'], all_visits[2:4])): url = reverse('api-1-origin-visits', url_args={'origin_url': new_origin['url']}, query_params={'per_page': 2, 'last_visit': last_visit}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') for expected_visit in expected_visits: origin_visit_url = reverse( 'api-1-origin-visit', url_args={'origin_url': new_origin['url'], 'visit_id': expected_visit['visit']}) snapshot_url = reverse( 'api-1-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_visit_url'] = origin_visit_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visits) @given(new_origin(), visit_dates(3), new_snapshots(3)) def test_api_lookup_origin_visit(self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( - new_origin['url'], visit_date) + new_origin['url'], visit_date, type='git') visit_id = origin_visit['visit'] self.storage.snapshot_add([new_snapshots[i]]) self.storage.origin_visit_update( new_origin['url'], origin_visit['visit'], snapshot=new_snapshots[i]['id']) url = reverse('api-1-origin-visit', url_args={'origin_url': new_origin['url'], 'visit_id': visit_id}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') expected_visit = self.origin_visit_get_by( new_origin['url'], visit_id) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) snapshot_url = reverse( 'api-1-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_url'] = origin_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visit) @given(new_origin(), visit_dates(2), new_snapshots(1)) def test_api_lookup_origin_visit_latest( self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) visit_dates.sort() visit_ids = [] for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( - new_origin['url'], visit_date) + new_origin['url'], visit_date, type='git') visit_ids.append(origin_visit['visit']) self.storage.snapshot_add([new_snapshots[0]]) self.storage.origin_visit_update( new_origin['url'], visit_ids[0], snapshot=new_snapshots[0]['id']) url = reverse('api-1-origin-visit-latest', url_args={'origin_url': new_origin['url']}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') expected_visit = self.origin_visit_get_by( new_origin['url'], visit_ids[1]) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_url'] = origin_url expected_visit['snapshot_url'] = None self.assertEqual(rv.data, expected_visit) @given(new_origin(), visit_dates(2), new_snapshots(1)) def test_api_lookup_origin_visit_latest_with_snapshot( self, new_origin, visit_dates, new_snapshots): self.storage.origin_add_one(new_origin) visit_dates.sort() visit_ids = [] for i, visit_date in enumerate(visit_dates): origin_visit = self.storage.origin_visit_add( - new_origin['url'], visit_date) + new_origin['url'], visit_date, type='git') visit_ids.append(origin_visit['visit']) self.storage.snapshot_add([new_snapshots[0]]) self.storage.origin_visit_update( new_origin['url'], visit_ids[0], snapshot=new_snapshots[0]['id']) url = reverse('api-1-origin-visit-latest', url_args={'origin_url': new_origin['url']}) url += '?require_snapshot=true' rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') expected_visit = self.origin_visit_get_by( new_origin['url'], visit_ids[0]) origin_url = reverse('api-1-origin', url_args={'origin_url': new_origin['url']}) snapshot_url = reverse( 'api-1-snapshot', url_args={'snapshot_id': expected_visit['snapshot']}) expected_visit['origin'] = new_origin['url'] expected_visit['origin_url'] = origin_url expected_visit['snapshot_url'] = snapshot_url self.assertEqual(rv.data, expected_visit) @given(origin()) def test_api_lookup_origin_visit_not_found(self, origin): all_visits = list(reversed(get_origin_visits(origin))) max_visit_id = max([v['visit'] for v in all_visits]) url = reverse('api-1-origin-visit', url_args={'origin_url': origin['url'], 'visit_id': max_visit_id + 1}) rv = self.client.get(url) self.assertEqual(rv.status_code, 404, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'NotFoundExc', 'reason': 'Origin %s or its visit with id %s not found!' % (origin['url'], max_visit_id+1) }) @pytest.mark.origin_id def test_api_origins(self): origins = get_tests_data()['origins'] origin_urls = {origin['url'] for origin in origins} # Get only one url = reverse('api-1-origins', query_params={'origin_count': 1}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) self.assertLess({origin['url'] for origin in rv.data}, origin_urls) # Get all url = reverse('api-1-origins', query_params={'origin_count': len(origins)}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), len(origins)) self.assertEqual({origin['url'] for origin in rv.data}, origin_urls) # Get "all + 10" url = reverse('api-1-origins', query_params={'origin_count': len(origins)+10}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), len(origins)) self.assertEqual({origin['url'] for origin in rv.data}, origin_urls) @pytest.mark.origin_id @given(strategies.integers(min_value=1)) def test_api_origins_scroll(self, origin_count): origins = get_tests_data()['origins'] origin_urls = {origin['url'] for origin in origins} url = reverse('api-1-origins', query_params={'origin_count': origin_count}) results = self._scroll_results(url) self.assertEqual(len(results), len(origins)) self.assertEqual({origin['url'] for origin in results}, origin_urls) @given(origin()) def test_api_origin_by_url(self, origin): url = reverse('api-1-origin', url_args={'origin_url': origin['url']}) rv = self.client.get(url) expected_origin = self.origin_get(origin) origin_visits_url = reverse('api-1-origin-visits', url_args={'origin_url': origin['url']}) expected_origin['origin_visits_url'] = origin_visits_url self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, expected_origin) - @given(origin()) - def test_api_origin_by_type_url(self, origin): - - url = reverse('api-1-origin', - url_args={'origin_type': origin['type'], - 'origin_url': origin['url']}) - rv = self.client.get(url) - - expected_origin = self.origin_get(origin) - - origin_visits_url = reverse('api-1-origin-visits', - url_args={'origin_url': origin['url']}) - - expected_origin['origin_visits_url'] = origin_visits_url - - self.assertEqual(rv.status_code, 200, rv.data) - self.assertEqual(rv['Content-Type'], 'application/json') - self.assertEqual(rv.data, expected_origin) - @given(new_origin()) def test_api_origin_not_found(self, new_origin): url = reverse('api-1-origin', - url_args={'origin_type': new_origin['type'], - 'origin_url': new_origin['url']}) + url_args={'origin_url': new_origin['url']}) rv = self.client.get(url) self.assertEqual(rv.status_code, 404, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(rv.data, { 'exception': 'NotFoundExc', - 'reason': 'Origin %s not found!' % new_origin['url'] + 'reason': 'Origin with url %s not found!' % new_origin['url'] }) @pytest.mark.origin_id def test_api_origin_search(self): expected_origins = { 'https://github.com/wcoder/highlightjs-line-numbers.js', 'https://github.com/memononen/libtess2', } # Search for 'github.com', get only one url = reverse('api-1-origin-search', url_args={'url_pattern': 'github.com'}, query_params={'limit': 1}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) self.assertLess({origin['url'] for origin in rv.data}, expected_origins) # Search for 'github.com', get all url = reverse('api-1-origin-search', url_args={'url_pattern': 'github.com'}, query_params={'limit': 2}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual({origin['url'] for origin in rv.data}, expected_origins) # Search for 'github.com', get more than available url = reverse('api-1-origin-search', url_args={'url_pattern': 'github.com'}, query_params={'limit': 10}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual({origin['url'] for origin in rv.data}, expected_origins) @pytest.mark.origin_id def test_api_origin_search_regexp(self): expected_origins = { 'https://github.com/memononen/libtess2', 'repo_with_submodules' } url = reverse('api-1-origin-search', url_args={'url_pattern': '(repo|libtess)'}, query_params={'limit': 10, 'regexp': True}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.data) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual({origin['url'] for origin in rv.data}, expected_origins) @pytest.mark.origin_id @given(strategies.integers(min_value=1)) def test_api_origin_search_scroll(self, limit): expected_origins = { 'https://github.com/wcoder/highlightjs-line-numbers.js', 'https://github.com/memononen/libtess2', } url = reverse('api-1-origin-search', url_args={'url_pattern': 'github.com'}, query_params={'limit': limit}) results = self._scroll_results(url) self.assertEqual({origin['url'] for origin in results}, expected_origins) @given(origin()) def test_api_origin_metadata_search(self, origin): with patch('swh.web.common.service.idx_storage') as mock_idx_storage: mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .side_effect = lambda conjunction, limit: [{ 'from_revision': ( b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' b'\xf2U\xfa\x05B8'), 'metadata': {'author': 'Jane Doe'}, 'origin_url': origin['url'], 'tool': { 'configuration': { 'context': ['NpmMapping', 'CodemetaMapping'], 'type': 'local' }, 'id': 3, 'name': 'swh-metadata-detector', 'version': '0.0.1' } }] url = reverse('api-1-origin-metadata-search', query_params={'fulltext': 'Jane Doe'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') expected_data = [{ - 'type': origin['type'], 'url': origin['url'], 'metadata': { 'metadata': {'author': 'Jane Doe'}, 'from_revision': ( '7026b7c1a2af56521e951c01ed20f255fa054238'), 'tool': { 'configuration': { 'context': ['NpmMapping', 'CodemetaMapping'], 'type': 'local' }, 'id': 3, 'name': 'swh-metadata-detector', 'version': '0.0.1', } } }] actual_data = rv.data for d in actual_data: if 'id' in d: del d['id'] self.assertEqual(rv.data, expected_data) mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=70) @given(origin()) def test_api_origin_metadata_search_limit(self, origin): with patch('swh.web.common.service.idx_storage') as mock_idx_storage: mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .side_effect = lambda conjunction, limit: [{ 'from_revision': ( b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' b'\xf2U\xfa\x05B8'), 'metadata': {'author': 'Jane Doe'}, 'origin_url': origin['url'], 'tool': { 'configuration': { 'context': ['NpmMapping', 'CodemetaMapping'], 'type': 'local' }, 'id': 3, 'name': 'swh-metadata-detector', 'version': '0.0.1' } }] url = reverse('api-1-origin-metadata-search', query_params={'fulltext': 'Jane Doe'}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=70) url = reverse('api-1-origin-metadata-search', query_params={'fulltext': 'Jane Doe', 'limit': 10}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=10) url = reverse('api-1-origin-metadata-search', query_params={'fulltext': 'Jane Doe', 'limit': 987}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') self.assertEqual(len(rv.data), 1) mock_idx_storage.origin_intrinsic_metadata_search_fulltext \ .assert_called_with(conjunction=['Jane Doe'], limit=100) @given(origin()) def test_api_origin_intrinsic_metadata(self, origin): with patch('swh.web.common.service.idx_storage') as mock_idx_storage: mock_idx_storage.origin_intrinsic_metadata_get \ .side_effect = lambda origin_urls: [{ 'from_revision': ( b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' b'\xf2U\xfa\x05B8'), 'metadata': {'author': 'Jane Doe'}, 'origin_url': origin['url'], 'tool': { 'configuration': { 'context': ['NpmMapping', 'CodemetaMapping'], 'type': 'local' }, 'id': 3, 'name': 'swh-metadata-detector', 'version': '0.0.1' } }] url = reverse('api-origin-intrinsic-metadata', - url_args={'origin_type': origin['type'], - 'origin_url': origin['url']}) + url_args={'origin_url': origin['url']}) rv = self.client.get(url) mock_idx_storage.origin_intrinsic_metadata_get \ .assert_called_once_with([origin['url']]) self.assertEqual(rv.status_code, 200, rv.content) self.assertEqual(rv['Content-Type'], 'application/json') expected_data = {'author': 'Jane Doe'} self.assertEqual(rv.data, expected_data) @patch('swh.web.common.service.idx_storage') def test_api_origin_metadata_search_invalid(self, mock_idx_storage): url = reverse('api-1-origin-metadata-search') rv = self.client.get(url) self.assertEqual(rv.status_code, 400, rv.content) mock_idx_storage.assert_not_called() diff --git a/swh/web/tests/api/views/test_origin_save.py b/swh/web/tests/api/views/test_origin_save.py index 3374ec37..91d3be7b 100644 --- a/swh/web/tests/api/views/test_origin_save.py +++ b/swh/web/tests/api/views/test_origin_save.py @@ -1,261 +1,261 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime, timedelta from django.utils import timezone from rest_framework.test import APITestCase from unittest.mock import patch from swh.web.common.utils import reverse from swh.web.common.models import ( SaveUnauthorizedOrigin, SaveOriginRequest, SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_REQUEST_PENDING ) from swh.web.common.models import ( SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_SCHEDULED, SAVE_TASK_FAILED, SAVE_TASK_SUCCEED ) from swh.web.tests.testcase import WebTestCase class SaveApiTestCase(WebTestCase, APITestCase): @classmethod def setUpTestData(cls): # noqa: N802 SaveUnauthorizedOrigin.objects.create( url='https://github.com/user/illegal_repo') SaveUnauthorizedOrigin.objects.create( url='https://gitlab.com/user_to_exclude') - def test_invalid_origin_type(self): + def test_invalid_visit_type(self): url = reverse('api-1-save-origin', - url_args={'origin_type': 'foo', + url_args={'visit_type': 'foo', 'origin_url': 'https://github.com/torvalds/linux'}) # noqa response = self.client.post(url) self.assertEqual(response.status_code, 400) def test_invalid_origin_url(self): url = reverse('api-1-save-origin', - url_args={'origin_type': 'git', + url_args={'visit_type': 'git', 'origin_url': 'bar'}) response = self.client.post(url) self.assertEqual(response.status_code, 400) def check_created_save_request_status(self, mock_scheduler, origin_url, scheduler_task_status, expected_request_status, expected_task_status=None, visit_date=None): if not scheduler_task_status: mock_scheduler.get_tasks.return_value = [] else: mock_scheduler.get_tasks.return_value = \ [{ 'priority': 'high', 'policy': 'oneshot', 'type': 'load-git', 'arguments': { 'kwargs': { 'repo_url': origin_url }, 'args': [] }, 'status': scheduler_task_status, 'id': 1, }] mock_scheduler.create_tasks.return_value = \ [{ 'priority': 'high', 'policy': 'oneshot', 'type': 'load-git', 'arguments': { 'kwargs': { 'repo_url': origin_url }, 'args': [] }, 'status': 'next_run_not_scheduled', 'id': 1, }] url = reverse('api-1-save-origin', - url_args={'origin_type': 'git', + url_args={'visit_type': 'git', 'origin_url': origin_url}) with patch('swh.web.common.origin_save._get_visit_info_for_save_request') as mock_visit_date: # noqa mock_visit_date.return_value = (visit_date, None) response = self.client.post(url) if expected_request_status != SAVE_REQUEST_REJECTED: self.assertEqual(response.status_code, 200) self.assertEqual(response.data['save_request_status'], expected_request_status) self.assertEqual(response.data['save_task_status'], expected_task_status) else: self.assertEqual(response.status_code, 403) def check_save_request_status(self, mock_scheduler, origin_url, expected_request_status, expected_task_status, scheduler_task_status='next_run_not_scheduled', # noqa visit_date=None): mock_scheduler.get_tasks.return_value = \ [{ 'priority': 'high', 'policy': 'oneshot', 'type': 'load-git', 'arguments': { 'kwargs': { 'repo_url': origin_url }, 'args': [] }, 'status': scheduler_task_status, 'id': 1, }] url = reverse('api-1-save-origin', - url_args={'origin_type': 'git', + url_args={'visit_type': 'git', 'origin_url': origin_url}) with patch('swh.web.common.origin_save._get_visit_info_for_save_request') as mock_visit_date: # noqa mock_visit_date.return_value = (visit_date, None) response = self.client.get(url) self.assertEqual(response.status_code, 200) save_request_data = response.data[0] self.assertEqual(save_request_data['save_request_status'], expected_request_status) self.assertEqual(save_request_data['save_task_status'], expected_task_status) # Check that save task status is still available when # the scheduler task has been archived mock_scheduler.get_tasks.return_value = [] response = self.client.get(url) self.assertEqual(response.status_code, 200) save_request_data = response.data[0] self.assertEqual(save_request_data['save_task_status'], expected_task_status) @patch('swh.web.common.origin_save.scheduler') def test_save_request_rejected(self, mock_scheduler): origin_url = 'https://github.com/user/illegal_repo' self.check_created_save_request_status(mock_scheduler, origin_url, None, SAVE_REQUEST_REJECTED) self.check_save_request_status(mock_scheduler, origin_url, SAVE_REQUEST_REJECTED, SAVE_TASK_NOT_CREATED) @patch('swh.web.common.origin_save.scheduler') def test_save_request_pending(self, mock_scheduler): origin_url = 'https://unkwownforge.com/user/repo' self.check_created_save_request_status(mock_scheduler, origin_url, None, SAVE_REQUEST_PENDING, SAVE_TASK_NOT_CREATED) self.check_save_request_status(mock_scheduler, origin_url, SAVE_REQUEST_PENDING, SAVE_TASK_NOT_CREATED) @patch('swh.web.common.origin_save.scheduler') def test_save_request_succeed(self, mock_scheduler): origin_url = 'https://github.com/Kitware/CMake' self.check_created_save_request_status(mock_scheduler, origin_url, None, SAVE_REQUEST_ACCEPTED, SAVE_TASK_NOT_YET_SCHEDULED) self.check_save_request_status(mock_scheduler, origin_url, SAVE_REQUEST_ACCEPTED, SAVE_TASK_SCHEDULED, scheduler_task_status='next_run_scheduled') # noqa self.check_save_request_status(mock_scheduler, origin_url, SAVE_REQUEST_ACCEPTED, SAVE_TASK_SUCCEED, scheduler_task_status='completed', visit_date=None) # noqa visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) self.check_save_request_status(mock_scheduler, origin_url, SAVE_REQUEST_ACCEPTED, SAVE_TASK_SUCCEED, scheduler_task_status='completed', visit_date=visit_date) # noqa @patch('swh.web.common.origin_save.scheduler') def test_save_request_failed(self, mock_scheduler): origin_url = 'https://gitlab.com/inkscape/inkscape' self.check_created_save_request_status(mock_scheduler, origin_url, None, SAVE_REQUEST_ACCEPTED, SAVE_TASK_NOT_YET_SCHEDULED) self.check_save_request_status(mock_scheduler, origin_url, SAVE_REQUEST_ACCEPTED, SAVE_TASK_SCHEDULED, scheduler_task_status='next_run_scheduled') # noqa self.check_save_request_status(mock_scheduler, origin_url, SAVE_REQUEST_ACCEPTED, SAVE_TASK_FAILED, scheduler_task_status='disabled') # noqa @patch('swh.web.common.origin_save.scheduler') def test_create_save_request_only_when_needed(self, mock_scheduler): origin_url = 'https://github.com/webpack/webpack' - SaveOriginRequest.objects.create(origin_type='git', + SaveOriginRequest.objects.create(visit_type='git', origin_url=origin_url, status=SAVE_REQUEST_ACCEPTED, # noqa loading_task_id=56) self.check_created_save_request_status(mock_scheduler, origin_url, 'next_run_not_scheduled', SAVE_REQUEST_ACCEPTED, SAVE_TASK_NOT_YET_SCHEDULED) - sors = list(SaveOriginRequest.objects.filter(origin_type='git', + sors = list(SaveOriginRequest.objects.filter(visit_type='git', origin_url=origin_url)) self.assertEqual(len(sors), 1) self.check_created_save_request_status(mock_scheduler, origin_url, 'next_run_scheduled', SAVE_REQUEST_ACCEPTED, SAVE_TASK_SCHEDULED) - sors = list(SaveOriginRequest.objects.filter(origin_type='git', + sors = list(SaveOriginRequest.objects.filter(visit_type='git', origin_url=origin_url)) self.assertEqual(len(sors), 1) visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) self.check_created_save_request_status(mock_scheduler, origin_url, 'completed', SAVE_REQUEST_ACCEPTED, SAVE_TASK_NOT_YET_SCHEDULED, visit_date=visit_date) - sors = list(SaveOriginRequest.objects.filter(origin_type='git', + sors = list(SaveOriginRequest.objects.filter(visit_type='git', origin_url=origin_url)) self.assertEqual(len(sors), 2) self.check_created_save_request_status(mock_scheduler, origin_url, 'disabled', SAVE_REQUEST_ACCEPTED, SAVE_TASK_NOT_YET_SCHEDULED) - sors = list(SaveOriginRequest.objects.filter(origin_type='git', + sors = list(SaveOriginRequest.objects.filter(visit_type='git', origin_url=origin_url)) self.assertEqual(len(sors), 3) def test_get_save_requests_unknown_origin(self): unknown_origin_url = 'https://gitlab.com/foo/bar' url = reverse('api-1-save-origin', - url_args={'origin_type': 'git', + url_args={'visit_type': 'git', 'origin_url': unknown_origin_url}) response = self.client.get(url) self.assertEqual(response.status_code, 404) self.assertEqual(response.data, { 'exception': 'NotFoundExc', - 'reason': ('No save requests found for origin with type ' - 'git and url %s.') % unknown_origin_url + 'reason': ('No save requests found for visit of type ' + 'git on origin with url %s.') % unknown_origin_url }) diff --git a/swh/web/tests/browse/views/test_identifiers.py b/swh/web/tests/browse/views/test_identifiers.py index 041366f1..e0297993 100644 --- a/swh/web/tests/browse/views/test_identifiers.py +++ b/swh/web/tests/browse/views/test_identifiers.py @@ -1,162 +1,159 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from hypothesis import given from swh.web.common.utils import reverse from swh.web.tests.strategies import ( content, directory, revision, release, snapshot ) from swh.web.tests.testcase import WebTestCase swh_id_prefix = 'swh:1:' class SwhBrowseIdTest(WebTestCase): @given(content()) def test_content_id_browse(self, content): cnt_sha1_git = content['sha1_git'] swh_id = swh_id_prefix + 'cnt:' + cnt_sha1_git url = reverse('browse-swh-id', url_args={'swh_id': swh_id}) query_string = 'sha1_git:' + cnt_sha1_git content_browse_url = reverse('browse-content', url_args={'query_string': query_string}) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], content_browse_url) @given(directory()) def test_directory_id_browse(self, directory): swh_id = swh_id_prefix + 'dir:' + directory url = reverse('browse-swh-id', url_args={'swh_id': swh_id}) directory_browse_url = reverse('browse-directory', url_args={'sha1_git': directory}) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], directory_browse_url) @given(revision()) def test_revision_id_browse(self, revision): swh_id = swh_id_prefix + 'rev:' + revision url = reverse('browse-swh-id', url_args={'swh_id': swh_id}) revision_browse_url = reverse('browse-revision', url_args={'sha1_git': revision}) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], revision_browse_url) - query_params = {'origin_type': 'git', - 'origin': 'https://github.com/user/repo'} + query_params = {'origin': 'https://github.com/user/repo'} url = reverse('browse-swh-id', url_args={'swh_id': swh_id}, query_params=query_params) revision_browse_url = reverse('browse-revision', url_args={'sha1_git': revision}, query_params=query_params) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], revision_browse_url) @given(release()) def test_release_id_browse(self, release): swh_id = swh_id_prefix + 'rel:' + release url = reverse('browse-swh-id', url_args={'swh_id': swh_id}) release_browse_url = reverse('browse-release', url_args={'sha1_git': release}) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], release_browse_url) - query_params = {'origin_type': 'git', - 'origin': 'https://github.com/user/repo'} + query_params = {'origin': 'https://github.com/user/repo'} url = reverse('browse-swh-id', url_args={'swh_id': swh_id}, query_params=query_params) release_browse_url = reverse('browse-release', url_args={'sha1_git': release}, query_params=query_params) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], release_browse_url) @given(snapshot()) def test_snapshot_id_browse(self, snapshot): swh_id = swh_id_prefix + 'snp:' + snapshot url = reverse('browse-swh-id', url_args={'swh_id': swh_id}) snapshot_browse_url = reverse('browse-snapshot', url_args={'snapshot_id': snapshot}) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], snapshot_browse_url) - query_params = {'origin_type': 'git', - 'origin': 'https://github.com/user/repo'} + query_params = {'origin': 'https://github.com/user/repo'} url = reverse('browse-swh-id', url_args={'swh_id': swh_id}, query_params=query_params) release_browse_url = reverse('browse-snapshot', url_args={'snapshot_id': snapshot}, query_params=query_params) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], release_browse_url) @given(release()) def test_bad_id_browse(self, release): swh_id = swh_id_prefix + 'foo:' + release url = reverse('browse-swh-id', url_args={'swh_id': swh_id}) resp = self.client.get(url) self.assertEqual(resp.status_code, 400) @given(content()) def test_content_id_optional_parts_browse(self, content): cnt_sha1_git = content['sha1_git'] optional_parts = ';lines=4-20;origin=https://github.com/user/repo' swh_id = swh_id_prefix + 'cnt:' + cnt_sha1_git + optional_parts url = reverse('browse-swh-id', url_args={'swh_id': swh_id}) query_string = 'sha1_git:' + cnt_sha1_git content_browse_url = reverse( 'browse-content', url_args={'query_string': query_string}, query_params={'origin': 'https://github.com/user/repo'}) content_browse_url += '#L4-L20' resp = self.client.get(url) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['location'], content_browse_url) diff --git a/swh/web/tests/browse/views/test_origin.py b/swh/web/tests/browse/views/test_origin.py index c0242aa7..aa072946 100644 --- a/swh/web/tests/browse/views/test_origin.py +++ b/swh/web/tests/browse/views/test_origin.py @@ -1,933 +1,914 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import random from unittest.mock import patch from django.utils.html import escape from hypothesis import given from swh.model.hashutil import hash_to_bytes from swh.web.browse.utils import process_snapshot_branches from swh.web.common.exc import NotFoundExc from swh.web.common.utils import ( reverse, gen_path_info, format_utc_iso_date, parse_timestamp, get_swh_persistent_id ) from swh.web.tests.data import get_content from swh.web.tests.strategies import ( origin, origin_with_multiple_visits, new_origin, new_snapshot, visit_dates, revisions ) from swh.web.tests.testcase import WebTestCase class SwhBrowseOriginTest(WebTestCase): @given(origin_with_multiple_visits()) def test_origin_visits_browse(self, origin): url = reverse('browse-origin-visits', - url_args={'origin_type': origin['type'], - 'origin_url': origin['url']}) + url_args={'origin_url': origin['url']}) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('origin-visits.html') url = reverse('browse-origin-visits', url_args={'origin_url': origin['url']}) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('origin-visits.html') visits = self.origin_visit_get(origin['url']) for v in visits: vdate = format_utc_iso_date(v['date'], '%Y-%m-%dT%H:%M:%SZ') browse_dir_url = reverse('browse-origin-directory', url_args={'origin_url': origin['url'], 'timestamp': vdate}) self.assertContains(resp, browse_dir_url) def origin_content_view_helper(self, origin_info, origin_visits, origin_branches, origin_releases, root_dir_sha1, content, visit_id=None, timestamp=None): content_path = '/'.join(content['path'].split('/')[1:]) - url_args = {'origin_type': origin_info['type'], - 'origin_url': origin_info['url'], + url_args = {'origin_url': origin_info['url'], 'path': content_path} if not visit_id: visit_id = origin_visits[-1]['visit'] query_params = {} if timestamp: url_args['timestamp'] = timestamp if visit_id: query_params['visit_id'] = visit_id url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('content.html') self.assertContains(resp, '' % content['hljs_language']) self.assertContains(resp, escape(content['data'])) split_path = content_path.split('/') filename = split_path[-1] path = content_path.replace(filename, '')[:-1] path_info = gen_path_info(path) del url_args['path'] if timestamp: url_args['timestamp'] = \ format_utc_iso_date(parse_timestamp(timestamp).isoformat(), '%Y-%m-%dT%H:%M:%S') root_dir_url = reverse('browse-origin-directory', url_args=url_args, query_params=query_params) self.assertContains(resp, '
  • ', count=len(path_info)+1) self.assertContains(resp, '%s' % (root_dir_url, root_dir_sha1[:7])) for p in path_info: url_args['path'] = p['path'] dir_url = reverse('browse-origin-directory', url_args=url_args, query_params=query_params) self.assertContains(resp, '%s' % (dir_url, p['name'])) self.assertContains(resp, '
  • %s
    • ' % filename) query_string = 'sha1_git:' + content['sha1_git'] url_raw = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params={'filename': filename}) self.assertContains(resp, url_raw) if 'args' in url_args: del url_args['path'] origin_branches_url = reverse('browse-origin-branches', url_args=url_args, query_params=query_params) self.assertContains(resp, 'Branches (%s)' % (origin_branches_url, len(origin_branches))) origin_releases_url = reverse('browse-origin-releases', url_args=url_args, query_params=query_params) self.assertContains(resp, 'Releases (%s)' % (origin_releases_url, len(origin_releases))) self.assertContains(resp, '
  • ', count=len(origin_branches)) url_args['path'] = content_path for branch in origin_branches: query_params['branch'] = branch['name'] root_dir_branch_url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) self.assertContains(resp, '' % root_dir_branch_url) self.assertContains(resp, '
  • ', count=len(origin_releases)) query_params['branch'] = None for release in origin_releases: query_params['release'] = release['name'] root_dir_release_url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) self.assertContains(resp, '' % root_dir_release_url) - del url_args['origin_type'] - url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('content.html') swh_cnt_id = get_swh_persistent_id('content', content['sha1_git']) swh_cnt_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_cnt_id}) self.assertContains(resp, swh_cnt_id) self.assertContains(resp, swh_cnt_id_url) self.assertContains(resp, 'swh-take-new-snapshot') @given(origin_with_multiple_visits()) def test_origin_content_view(self, origin): origin_visits = self.origin_visit_get(origin['url']) def _get_test_data(visit_idx): snapshot = self.snapshot_get(origin_visits[visit_idx]['snapshot']) head_rev_id = snapshot['branches']['HEAD']['target'] head_rev = self.revision_get(head_rev_id) dir_content = self.directory_ls(head_rev['directory']) dir_files = [e for e in dir_content if e['type'] == 'file'] dir_file = random.choice(dir_files) branches, releases = process_snapshot_branches(snapshot) return { 'branches': branches, 'releases': releases, 'root_dir_sha1': head_rev['directory'], 'content': get_content(dir_file['checksums']['sha1']), 'visit': origin_visits[visit_idx] } test_data = _get_test_data(-1) self.origin_content_view_helper(origin, origin_visits, test_data['branches'], test_data['releases'], test_data['root_dir_sha1'], test_data['content']) self.origin_content_view_helper(origin, origin_visits, test_data['branches'], test_data['releases'], test_data['root_dir_sha1'], test_data['content'], timestamp=test_data['visit']['date']) visit_unix_ts = parse_timestamp(test_data['visit']['date']).timestamp() visit_unix_ts = int(visit_unix_ts) self.origin_content_view_helper(origin, origin_visits, test_data['branches'], test_data['releases'], test_data['root_dir_sha1'], test_data['content'], timestamp=visit_unix_ts) test_data = _get_test_data(0) self.origin_content_view_helper(origin, origin_visits, test_data['branches'], test_data['releases'], test_data['root_dir_sha1'], test_data['content'], visit_id=test_data['visit']['visit']) def origin_directory_view_helper(self, origin_info, origin_visits, origin_branches, origin_releases, root_directory_sha1, directory_entries, visit_id=None, timestamp=None, path=None): dirs = [e for e in directory_entries if e['type'] in ('dir', 'rev')] files = [e for e in directory_entries if e['type'] == 'file'] if not visit_id: visit_id = origin_visits[-1]['visit'] url_args = {'origin_url': origin_info['url']} query_params = {} if timestamp: url_args['timestamp'] = timestamp else: query_params['visit_id'] = visit_id if path: url_args['path'] = path url = reverse('browse-origin-directory', url_args=url_args, query_params=query_params) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('directory.html') self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('directory.html') self.assertContains(resp, '', count=len(dirs)) self.assertContains(resp, '', count=len(files)) if timestamp: url_args['timestamp'] = \ format_utc_iso_date(parse_timestamp(timestamp).isoformat(), '%Y-%m-%dT%H:%M:%S') for d in dirs: if d['type'] == 'rev': dir_url = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: dir_path = d['name'] if path: dir_path = "%s/%s" % (path, d['name']) dir_url_args = dict(url_args) dir_url_args['path'] = dir_path dir_url = reverse('browse-origin-directory', url_args=dir_url_args, query_params=query_params) self.assertContains(resp, dir_url) for f in files: file_path = f['name'] if path: file_path = "%s/%s" % (path, f['name']) file_url_args = dict(url_args) file_url_args['path'] = file_path file_url = reverse('browse-origin-content', url_args=file_url_args, query_params=query_params) self.assertContains(resp, file_url) if 'path' in url_args: del url_args['path'] root_dir_branch_url = \ reverse('browse-origin-directory', url_args=url_args, query_params=query_params) nb_bc_paths = 1 if path: nb_bc_paths = len(path.split('/')) + 1 self.assertContains(resp, '
  • ', count=nb_bc_paths) self.assertContains(resp, '%s' % (root_dir_branch_url, root_directory_sha1[:7])) origin_branches_url = reverse('browse-origin-branches', url_args=url_args, query_params=query_params) self.assertContains(resp, 'Branches (%s)' % (origin_branches_url, len(origin_branches))) origin_releases_url = reverse('browse-origin-releases', url_args=url_args, query_params=query_params) nb_releases = len(origin_releases) if nb_releases > 0: self.assertContains(resp, 'Releases (%s)' % (origin_releases_url, nb_releases)) if path: url_args['path'] = path self.assertContains(resp, '
  • ', count=len(origin_branches)) for branch in origin_branches: query_params['branch'] = branch['name'] root_dir_branch_url = \ reverse('browse-origin-directory', url_args=url_args, query_params=query_params) self.assertContains(resp, '' % root_dir_branch_url) self.assertContains(resp, '
  • ', count=len(origin_releases)) query_params['branch'] = None for release in origin_releases: query_params['release'] = release['name'] root_dir_release_url = \ reverse('browse-origin-directory', url_args=url_args, query_params=query_params) self.assertContains(resp, '' % root_dir_release_url) self.assertContains(resp, 'vault-cook-directory') self.assertContains(resp, 'vault-cook-revision') swh_dir_id = get_swh_persistent_id('directory', directory_entries[0]['dir_id']) # noqa swh_dir_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_dir_id}) self.assertContains(resp, swh_dir_id) self.assertContains(resp, swh_dir_id_url) self.assertContains(resp, 'swh-take-new-snapshot') @given(origin()) def test_origin_root_directory_view(self, origin): origin_visits = self.origin_visit_get(origin['url']) visit = origin_visits[-1] snapshot = self.snapshot_get(visit['snapshot']) head_rev_id = snapshot['branches']['HEAD']['target'] head_rev = self.revision_get(head_rev_id) root_dir_sha1 = head_rev['directory'] dir_content = self.directory_ls(root_dir_sha1) branches, releases = process_snapshot_branches(snapshot) visit_unix_ts = parse_timestamp(visit['date']).timestamp() visit_unix_ts = int(visit_unix_ts) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, dir_content) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, dir_content, visit_id=visit['visit']) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, dir_content, timestamp=visit_unix_ts) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, dir_content, timestamp=visit['date']) origin = dict(origin) del origin['type'] self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, dir_content) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, dir_content, visit_id=visit['visit']) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, dir_content, timestamp=visit_unix_ts) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, dir_content, timestamp=visit['date']) @given(origin()) def test_origin_sub_directory_view(self, origin): origin_visits = self.origin_visit_get(origin['url']) visit = origin_visits[-1] snapshot = self.snapshot_get(visit['snapshot']) head_rev_id = snapshot['branches']['HEAD']['target'] head_rev = self.revision_get(head_rev_id) root_dir_sha1 = head_rev['directory'] subdirs = [e for e in self.directory_ls(root_dir_sha1) if e['type'] == 'dir'] branches, releases = process_snapshot_branches(snapshot) visit_unix_ts = parse_timestamp(visit['date']).timestamp() visit_unix_ts = int(visit_unix_ts) if len(subdirs) == 0: return subdir = random.choice(subdirs) subdir_content = self.directory_ls(subdir['target']) subdir_path = subdir['name'] self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, subdir_content, path=subdir_path) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, subdir_content, path=subdir_path, visit_id=visit['visit']) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, subdir_content, path=subdir_path, timestamp=visit_unix_ts) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, subdir_content, path=subdir_path, timestamp=visit['date']) origin = dict(origin) del origin['type'] self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, subdir_content, path=subdir_path) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, subdir_content, path=subdir_path, visit_id=visit['visit']) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, subdir_content, path=subdir_path, timestamp=visit_unix_ts) self.origin_directory_view_helper(origin, origin_visits, branches, releases, root_dir_sha1, subdir_content, path=subdir_path, timestamp=visit['date']) def origin_branches_helper(self, origin_info, origin_snapshot): - url_args = {'origin_type': origin_info['type'], - 'origin_url': origin_info['url']} + url_args = {'origin_url': origin_info['url']} url = reverse('browse-origin-branches', url_args=url_args) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('branches.html') origin_branches = origin_snapshot[0] origin_releases = origin_snapshot[1] origin_branches_url = reverse('browse-origin-branches', url_args=url_args) self.assertContains(resp, 'Branches (%s)' % (origin_branches_url, len(origin_branches))) origin_releases_url = reverse('browse-origin-releases', url_args=url_args) nb_releases = len(origin_releases) if nb_releases > 0: self.assertContains(resp, 'Releases (%s)' % (origin_releases_url, nb_releases)) self.assertContains(resp, '' % escape(browse_branch_url)) browse_revision_url = reverse( 'browse-revision', url_args={'sha1_git': branch['revision']}, - query_params={'origin_type': origin_info['type'], - 'origin': origin_info['url']}) + query_params={'origin': origin_info['url']}) self.assertContains(resp, '' % escape(browse_revision_url)) @given(origin()) def test_origin_branches(self, origin): origin_visits = self.origin_visit_get(origin['url']) visit = origin_visits[-1] snapshot = self.snapshot_get(visit['snapshot']) snapshot_content = process_snapshot_branches(snapshot) self.origin_branches_helper(origin, snapshot_content) origin = dict(origin) origin['type'] = None self.origin_branches_helper(origin, snapshot_content) def origin_releases_helper(self, origin_info, origin_snapshot): - url_args = {'origin_type': origin_info['type'], - 'origin_url': origin_info['url']} + url_args = {'origin_url': origin_info['url']} url = reverse('browse-origin-releases', url_args=url_args) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('releases.html') origin_branches = origin_snapshot[0] origin_releases = origin_snapshot[1] origin_branches_url = reverse('browse-origin-branches', url_args=url_args) self.assertContains(resp, 'Branches (%s)' % (origin_branches_url, len(origin_branches))) origin_releases_url = reverse('browse-origin-releases', url_args=url_args) nb_releases = len(origin_releases) if nb_releases > 0: self.assertContains(resp, 'Releases (%s)' % (origin_releases_url, nb_releases)) self.assertContains(resp, '' % escape(browse_release_url)) self.assertContains(resp, '' % escape(browse_revision_url)) @given(origin()) def test_origin_releases(self, origin): origin_visits = self.origin_visit_get(origin['url']) visit = origin_visits[-1] snapshot = self.snapshot_get(visit['snapshot']) snapshot_content = process_snapshot_branches(snapshot) self.origin_releases_helper(origin, snapshot_content) origin = dict(origin) origin['type'] = None self.origin_releases_helper(origin, snapshot_content) @given(new_origin(), new_snapshot(min_size=4, max_size=4), visit_dates(), revisions(min_size=3, max_size=3)) def test_origin_snapshot_null_branch(self, new_origin, new_snapshot, visit_dates, revisions): snp_dict = new_snapshot.to_dict() new_origin = self.storage.origin_add([new_origin])[0] for i, branch in enumerate(snp_dict['branches'].keys()): if i == 0: snp_dict['branches'][branch] = None else: snp_dict['branches'][branch] = { 'target_type': 'revision', 'target': hash_to_bytes(revisions[i-1]), } self.storage.snapshot_add([snp_dict]) visit = self.storage.origin_visit_add( - new_origin['url'], visit_dates[0]) + new_origin['url'], visit_dates[0], type='git') self.storage.origin_visit_update(new_origin['url'], visit['visit'], status='partial', snapshot=snp_dict['id']) url = reverse('browse-origin-directory', url_args={'origin_url': new_origin['url']}) rv = self.client.get(url) self.assertEqual(rv.status_code, 200) @given(new_origin(), new_snapshot(min_size=4, max_size=4), visit_dates(), revisions(min_size=3, max_size=3)) def test_origin_snapshot_invalid_branch(self, new_origin, new_snapshot, visit_dates, revisions): snp_dict = new_snapshot.to_dict() new_origin = self.storage.origin_add([new_origin])[0] for i, branch in enumerate(snp_dict['branches'].keys()): if i == 0: invalid_branch = branch else: snp_dict['branches'][branch] = { 'target_type': 'revision', 'target': hash_to_bytes(revisions[i-1]), } del snp_dict['branches'][invalid_branch] self.storage.snapshot_add([snp_dict]) visit = self.storage.origin_visit_add( - new_origin['url'], visit_dates[0]) + new_origin['url'], visit_dates[0], type='git') self.storage.origin_visit_update(new_origin['url'], visit['visit'], status='full', snapshot=snp_dict['id']) url = reverse('browse-origin-directory', url_args={'origin_url': new_origin['url']}, query_params={'branch': invalid_branch}) rv = self.client.get(url) self.assertEqual(rv.status_code, 404) @patch('swh.web.browse.views.utils.snapshot_context.request_content') @patch('swh.web.common.origin_visits.get_origin_visits') @patch('swh.web.browse.utils.get_origin_visit_snapshot') @patch('swh.web.browse.utils.service') @patch('swh.web.browse.views.origin.service') @patch('swh.web.browse.views.utils.snapshot_context.service') - @patch('swh.web.browse.views.origin.get_origin_info') - def test_origin_request_errors(self, mock_get_origin_info, + def test_origin_request_errors(self, mock_snapshot_service, mock_origin_service, mock_utils_service, mock_get_origin_visit_snapshot, mock_get_origin_visits, mock_request_content): - mock_get_origin_info.side_effect = \ - NotFoundExc('origin not found') + mock_origin_service.lookup_origin.side_effect = NotFoundExc( + 'origin not found') url = reverse('browse-origin-visits', - url_args={'origin_type': 'foo', - 'origin_url': 'bar'}) + url_args={'origin_url': 'bar'}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertContains(resp, 'origin not found', status_code=404) - mock_utils_service.lookup_origin.side_effect = None - mock_utils_service.lookup_origin.return_value = {'type': 'foo', - 'url': 'bar', - 'id': 457} + mock_origin_service.lookup_origin.side_effect = None + mock_origin_service.lookup_origin.return_value = {'type': 'foo', + 'url': 'bar', + 'id': 457} mock_get_origin_visits.return_value = [] url = reverse('browse-origin-directory', - url_args={'origin_type': 'foo', - 'origin_url': 'bar'}) + url_args={'origin_url': 'bar'}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertContains(resp, "No visit", status_code=404) mock_get_origin_visits.return_value = [{'visit': 1}] mock_get_origin_visit_snapshot.side_effect = \ NotFoundExc('visit not found') url = reverse('browse-origin-directory', - url_args={'origin_type': 'foo', - 'origin_url': 'bar'}, + url_args={'origin_url': 'bar'}, query_params={'visit_id': 2}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertRegex(resp.content.decode('utf-8'), 'Visit.*not found') mock_get_origin_visits.return_value = [{ 'date': '2015-09-26T09:30:52.373449+00:00', 'metadata': {}, 'origin': 457, 'snapshot': 'bdaf9ac436488a8c6cda927a0f44e172934d3f65', 'status': 'full', 'visit': 1 }] mock_get_origin_visit_snapshot.side_effect = None mock_get_origin_visit_snapshot.return_value = ( [{'directory': 'ae59ceecf46367e8e4ad800e231fc76adc3afffb', 'name': 'HEAD', 'revision': '7bc08e1aa0b08cb23e18715a32aa38517ad34672', 'date': '04 May 2017, 13:27 UTC', 'message': ''}], [] ) mock_utils_service.lookup_snapshot_size.return_value = { 'revision': 1, 'release': 0 } mock_utils_service.lookup_directory.side_effect = \ NotFoundExc('Directory not found') url = reverse('browse-origin-directory', - url_args={'origin_type': 'foo', - 'origin_url': 'bar'}) + url_args={'origin_url': 'bar'}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertContains(resp, 'Directory not found', status_code=404) with patch('swh.web.browse.views.utils.snapshot_context.' 'get_snapshot_context') as mock_get_snapshot_context: mock_get_snapshot_context.side_effect = \ NotFoundExc('Snapshot not found') url = reverse('browse-origin-directory', - url_args={'origin_type': 'foo', - 'origin_url': 'bar'}) + url_args={'origin_url': 'bar'}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertContains(resp, 'Snapshot not found', status_code=404) mock_origin_service.lookup_origin.side_effect = None mock_origin_service.lookup_origin.return_value = {'type': 'foo', 'url': 'bar', 'id': 457} mock_get_origin_visits.return_value = [] url = reverse('browse-origin-content', - url_args={'origin_type': 'foo', - 'origin_url': 'bar', + url_args={'origin_url': 'bar', 'path': 'foo'}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertContains(resp, "No visit", status_code=404) mock_get_origin_visits.return_value = [{'visit': 1}] mock_get_origin_visit_snapshot.side_effect = \ NotFoundExc('visit not found') url = reverse('browse-origin-content', - url_args={'origin_type': 'foo', - 'origin_url': 'bar', + url_args={'origin_url': 'bar', 'path': 'foo'}, query_params={'visit_id': 2}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertRegex(resp.content.decode('utf-8'), 'Visit.*not found') mock_get_origin_visits.return_value = [{ 'date': '2015-09-26T09:30:52.373449+00:00', 'metadata': {}, 'origin': 457, 'snapshot': 'bdaf9ac436488a8c6cda927a0f44e172934d3f65', 'status': 'full', 'visit': 1 }] mock_get_origin_visit_snapshot.side_effect = None mock_get_origin_visit_snapshot.return_value = ([], []) url = reverse('browse-origin-content', - url_args={'origin_type': 'foo', - 'origin_url': 'bar', + url_args={'origin_url': 'bar', 'path': 'baz'}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertRegex(resp.content.decode('utf-8'), 'Origin.*has an empty list of branches') mock_get_origin_visit_snapshot.return_value = ( [{'directory': 'ae59ceecf46367e8e4ad800e231fc76adc3afffb', 'name': 'HEAD', 'revision': '7bc08e1aa0b08cb23e18715a32aa38517ad34672', 'date': '04 May 2017, 13:27 UTC', 'message': ''}], [] ) mock_snapshot_service.lookup_directory_with_path.return_value = \ {'target': '5ecd9f37b7a2d2e9980d201acd6286116f2ba1f1'} mock_request_content.side_effect = \ NotFoundExc('Content not found') url = reverse('browse-origin-content', - url_args={'origin_type': 'foo', - 'origin_url': 'bar', + url_args={'origin_url': 'bar', 'path': 'baz'}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertContains(resp, 'Content not found', status_code=404) @patch('swh.web.common.origin_visits.get_origin_visits') @patch('swh.web.browse.utils.get_origin_visit_snapshot') @patch('swh.web.browse.utils.service') def test_origin_empty_snapshot(self, mock_utils_service, mock_get_origin_visit_snapshot, mock_get_origin_visits): mock_get_origin_visits.return_value = [{ 'date': '2015-09-26T09:30:52.373449+00:00', 'metadata': {}, 'origin': 457, 'snapshot': 'bdaf9ac436488a8c6cda927a0f44e172934d3f65', 'status': 'full', + 'type': 'git', 'visit': 1 }] mock_get_origin_visit_snapshot.return_value = ([], []) mock_utils_service.lookup_snapshot_size.return_value = { 'revision': 0, 'release': 0 } mock_utils_service.lookup_origin.return_value = { 'id': 457, - 'type': 'git', 'url': 'https://github.com/foo/bar' } url = reverse('browse-origin-directory', - url_args={'origin_type': 'foo', - 'origin_url': 'bar'}) + url_args={'origin_url': 'bar'}) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('content.html') self.assertRegex(resp.content.decode('utf-8'), 'snapshot.*is empty') diff --git a/swh/web/tests/browse/views/test_revision.py b/swh/web/tests/browse/views/test_revision.py index 9dc7c0fc..7856e8e4 100644 --- a/swh/web/tests/browse/views/test_revision.py +++ b/swh/web/tests/browse/views/test_revision.py @@ -1,247 +1,246 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.utils.html import escape from hypothesis import given from swh.web.common.utils import ( reverse, format_utc_iso_date, get_swh_persistent_id, parse_timestamp ) from swh.web.tests.strategies import ( origin, revision, unknown_revision, new_origin ) from swh.web.tests.testcase import WebTestCase class SwhBrowseRevisionTest(WebTestCase): @given(revision()) def test_revision_browse(self, revision): url = reverse('browse-revision', url_args={'sha1_git': revision}) revision_data = self.revision_get(revision) author_name = revision_data['author']['name'] committer_name = revision_data['committer']['name'] dir_id = revision_data['directory'] directory_url = reverse('browse-directory', url_args={'sha1_git': dir_id}) history_url = reverse('browse-revision-log', url_args={'sha1_git': revision}) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/revision.html') self.assertContains(resp, author_name) self.assertContains(resp, committer_name) self.assertContains(resp, directory_url) self.assertContains(resp, history_url) for parent in revision_data['parents']: parent_url = reverse('browse-revision', url_args={'sha1_git': parent}) self.assertContains(resp, '%s' % (parent_url, parent)) author_date = revision_data['date'] committer_date = revision_data['committer_date'] message_lines = revision_data['message'].split('\n') self.assertContains(resp, format_utc_iso_date(author_date)) self.assertContains(resp, format_utc_iso_date(committer_date)) self.assertContains(resp, escape(message_lines[0])) self.assertContains(resp, escape('\n'.join(message_lines[1:]))) @given(origin()) def test_revision_origin_browse(self, origin): snapshot = self.snapshot_get_latest(origin['url']) revision = snapshot['branches']['HEAD']['target'] revision_data = self.revision_get(revision) dir_id = revision_data['directory'] origin_revision_log_url = reverse('browse-origin-log', url_args={'origin_url': origin['url']}, # noqa query_params={'revision': revision}) url = reverse('browse-revision', url_args={'sha1_git': revision}, query_params={'origin': origin['url']}) resp = self.client.get(url) self.assertContains(resp, origin_revision_log_url) for parent in revision_data['parents']: parent_url = reverse('browse-revision', url_args={'sha1_git': parent}, query_params={'origin': origin['url']}) self.assertContains(resp, '%s' % (parent_url, parent)) self.assertContains(resp, 'vault-cook-directory') self.assertContains(resp, 'vault-cook-revision') swh_rev_id = get_swh_persistent_id('revision', revision) swh_rev_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_rev_id}) self.assertContains(resp, swh_rev_id) self.assertContains(resp, swh_rev_id_url) swh_dir_id = get_swh_persistent_id('directory', dir_id) swh_dir_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_dir_id}) self.assertContains(resp, swh_dir_id) self.assertContains(resp, swh_dir_id_url) self.assertContains(resp, 'swh-take-new-snapshot') @given(revision()) def test_revision_log_browse(self, revision): per_page = 10 revision_log = self.revision_log(revision) revision_log_sorted = \ sorted(revision_log, key=lambda rev: -parse_timestamp( rev['committer_date']).timestamp()) url = reverse('browse-revision-log', url_args={'sha1_git': revision}, query_params={'per_page': per_page}) resp = self.client.get(url) next_page_url = reverse('browse-revision-log', url_args={'sha1_git': revision}, query_params={'offset': per_page, 'per_page': per_page}) nb_log_entries = per_page if len(revision_log_sorted) < per_page: nb_log_entries = len(revision_log_sorted) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/revision-log.html') self.assertContains(resp, 'Newer') if len(revision_log_sorted) > per_page: self.assertContains(resp, 'Older' % # noqa escape(next_page_url)) for log in revision_log_sorted[:per_page]: revision_url = reverse('browse-revision', url_args={'sha1_git': log['id']}) self.assertContains(resp, log['id'][:7]) self.assertContains(resp, log['author']['name']) self.assertContains(resp, format_utc_iso_date(log['date'])) self.assertContains(resp, escape(log['message'])) self.assertContains(resp, format_utc_iso_date(log['committer_date'])) # noqa self.assertContains(resp, revision_url) if len(revision_log_sorted) <= per_page: return resp = self.client.get(next_page_url) prev_page_url = reverse('browse-revision-log', url_args={'sha1_git': revision}, query_params={'per_page': per_page}) next_page_url = reverse('browse-revision-log', url_args={'sha1_git': revision}, query_params={'offset': 2 * per_page, 'per_page': per_page}) nb_log_entries = len(revision_log_sorted) - per_page if nb_log_entries > per_page: nb_log_entries = per_page self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/revision-log.html') self.assertContains(resp, 'Newer' % escape(prev_page_url)) if len(revision_log_sorted) > 2 * per_page: self.assertContains(resp, 'Older' % # noqa escape(next_page_url)) if len(revision_log_sorted) <= 2 * per_page: return resp = self.client.get(next_page_url) prev_page_url = reverse('browse-revision-log', url_args={'sha1_git': revision}, query_params={'offset': per_page, 'per_page': per_page}) next_page_url = reverse('browse-revision-log', url_args={'sha1_git': revision}, query_params={'offset': 3 * per_page, 'per_page': per_page}) nb_log_entries = len(revision_log_sorted) - 2 * per_page if nb_log_entries > per_page: nb_log_entries = per_page self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/revision-log.html') self.assertContains(resp, 'Newer' % escape(prev_page_url)) if len(revision_log_sorted) > 3 * per_page: self.assertContains(resp, 'Older' % # noqa escape(next_page_url)) @given(revision(), unknown_revision(), new_origin()) def test_revision_request_errors(self, revision, unknown_revision, new_origin): url = reverse('browse-revision', url_args={'sha1_git': unknown_revision}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertContains(resp, 'Revision with sha1_git %s not found' % unknown_revision, status_code=404) url = reverse('browse-revision', url_args={'sha1_git': revision}, - query_params={'origin_type': new_origin['type'], - 'origin': new_origin['url']}) + query_params={'origin': new_origin['url']}) resp = self.client.get(url) self.assertEqual(resp.status_code, 404) self.assertTemplateUsed('error.html') self.assertContains(resp, 'the origin mentioned in your request' ' appears broken', status_code=404) @given(revision()) def test_revision_uppercase(self, revision): url = reverse('browse-revision-uppercase-checksum', url_args={'sha1_git': revision.upper()}) resp = self.client.get(url) self.assertEqual(resp.status_code, 302) redirect_url = reverse('browse-revision', url_args={'sha1_git': revision}) self.assertEqual(resp['location'], redirect_url) diff --git a/swh/web/tests/common/test_origin_save.py b/swh/web/tests/common/test_origin_save.py index 118a79d9..4c140bfc 100644 --- a/swh/web/tests/common/test_origin_save.py +++ b/swh/web/tests/common/test_origin_save.py @@ -1,127 +1,127 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import os from datetime import datetime, timedelta, timezone import pytest import requests_mock from swh.web.common.models import ( SaveOriginRequest ) from swh.web.common.origin_save import get_save_origin_task_info from swh.web.config import get_config _RESOURCES_PATH = os.path.join(os.path.dirname(__file__), '../resources') _es_url = 'http://esnode1.internal.softwareheritage.org:9200' _es_workers_index_url = '%s/swh_workers-*' % _es_url def _get_save_origin_task_info_test(mocker, task_archived=False, es_available=True): swh_web_config = get_config() if es_available: swh_web_config.update({'es_workers_index_url': _es_workers_index_url}) else: swh_web_config.update({'es_workers_index_url': ''}) sor_id = 4473 SaveOriginRequest.objects.create( id=sor_id, request_date=datetime(2019, 8, 30, 23, 7, 3, 474294, tzinfo=timezone.utc), - origin_type='git', + visit_type='git', origin_url='https://gitlab.com/inkscape/inkscape', status='accepted', loading_task_id=203525448, visit_date=datetime(2019, 8, 30, 23, 18, 11, 54341, tzinfo=timezone.utc) ) mock_scheduler = mocker.patch('swh.web.common.origin_save.scheduler') task = { 'arguments': { 'args': [], 'kwargs': { 'repo_url': 'https://gitlab.com/inkscape/inkscape' } }, 'current_interval': timedelta(days=64), 'id': 203525448, 'next_run': datetime(2019, 8, 30, 23, 7, 1, 614823), 'policy': 'oneshot', 'priority': 'high', 'retries_left': 0, 'status': 'disabled', 'type': 'load-git' } if not task_archived else None mock_scheduler.get_tasks.return_value = [task] task_run = { 'backend_id': 'f00c712c-e820-41ce-a07c-9bf8df914205', 'ended': datetime(2019, 8, 30, 23, 18, 13, 770800), 'id': 654270631, 'metadata': {}, 'scheduled': datetime(2019, 8, 30, 23, 8, 34, 282021), 'started': None, 'status': 'failed', 'task': 203525448 } mock_scheduler.get_task_runs.return_value = [task_run] es_response = os.path.join(_RESOURCES_PATH, 'json/es_task_info_response.json') with open(es_response) as json_fd: es_response = json.load(json_fd) task_exec_data = es_response['hits']['hits'][-1]['_source'] with requests_mock.Mocker() as requests_mocker: requests_mocker.register_uri('POST', _es_workers_index_url+'/_search', json=es_response) sor_task_info = get_save_origin_task_info(sor_id) expected_result = { 'type': task['type'], 'arguments': task['arguments'], 'id': task['id'], 'backend_id': task_run['backend_id'], 'scheduled': task_run['scheduled'], 'ended': task_run['ended'], 'status': task_run['status'], } if not task_archived else {} if es_available and not task_archived: expected_result.update({ 'message': task_exec_data['message'], 'name': task_exec_data['swh_task_name'], 'worker': task_exec_data['hostname'] }) assert sor_task_info == expected_result @pytest.mark.django_db def test_get_save_origin_archived_task_info(mocker): _get_save_origin_task_info_test(mocker, task_archived=True) @pytest.mark.django_db def test_get_save_origin_task_info_with_es(mocker): _get_save_origin_task_info_test(mocker, es_available=True) @pytest.mark.django_db def test_get_save_origin_task_info_without_es(mocker): _get_save_origin_task_info_test(mocker, es_available=False) diff --git a/swh/web/tests/common/test_service.py b/swh/web/tests/common/test_service.py index bff5e645..05d98393 100644 --- a/swh/web/tests/common/test_service.py +++ b/swh/web/tests/common/test_service.py @@ -1,838 +1,836 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import itertools import pytest import random from collections import defaultdict from hypothesis import given from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.from_disk import DentryPerms from swh.web.common import service from swh.web.common.exc import BadInputExc, NotFoundExc from swh.web.tests.data import random_sha1, random_content from swh.web.tests.strategies import ( content, contents, unknown_contents, contents_with_ctags, origin, new_origin, visit_dates, directory, release, revision, unknown_revision, revisions, ancestor_revisions, non_ancestor_revisions, invalid_sha1, sha256, revision_with_submodules, empty_directory, new_revision ) from swh.web.tests.testcase import ( WebTestCase, ctags_json_missing, fossology_missing ) class ServiceTestCase(WebTestCase): @given(contents()) def test_lookup_multiple_hashes_all_present(self, contents): input_data = [] expected_output = [] for cnt in contents: input_data.append({'sha1': cnt['sha1']}) expected_output.append({'sha1': cnt['sha1'], 'found': True}) self.assertEqual(service.lookup_multiple_hashes(input_data), expected_output) @given(contents(), unknown_contents()) def test_lookup_multiple_hashes_some_missing(self, contents, unknown_contents): input_contents = list(itertools.chain(contents, unknown_contents)) random.shuffle(input_contents) input_data = [] expected_output = [] for cnt in input_contents: input_data.append({'sha1': cnt['sha1']}) expected_output.append({'sha1': cnt['sha1'], 'found': cnt in contents}) self.assertEqual(service.lookup_multiple_hashes(input_data), expected_output) def test_lookup_hash_does_not_exist(self): unknown_content_ = random_content() actual_lookup = service.lookup_hash('sha1_git:%s' % unknown_content_['sha1_git']) self.assertEqual(actual_lookup, {'found': None, 'algo': 'sha1_git'}) @given(content()) def test_lookup_hash_exist(self, content): actual_lookup = service.lookup_hash('sha1:%s' % content['sha1']) content_metadata = self.content_get_metadata(content['sha1']) self.assertEqual({'found': content_metadata, 'algo': 'sha1'}, actual_lookup) def test_search_hash_does_not_exist(self): unknown_content_ = random_content() actual_lookup = service.search_hash('sha1_git:%s' % unknown_content_['sha1_git']) self.assertEqual({'found': False}, actual_lookup) @given(content()) def test_search_hash_exist(self, content): actual_lookup = service.search_hash('sha1:%s' % content['sha1']) self.assertEqual({'found': True}, actual_lookup) @pytest.mark.skipif(ctags_json_missing, reason="requires ctags with json output support") @given(contents_with_ctags()) def test_lookup_content_ctags(self, contents_with_ctags): content_sha1 = random.choice(contents_with_ctags['sha1s']) self.content_add_ctags(content_sha1) actual_ctags = \ list(service.lookup_content_ctags('sha1:%s' % content_sha1)) expected_data = list(self.content_get_ctags(content_sha1)) for ctag in expected_data: ctag['id'] = content_sha1 self.assertEqual(actual_ctags, expected_data) def test_lookup_content_ctags_no_hash(self): unknown_content_ = random_content() actual_ctags = \ list(service.lookup_content_ctags('sha1:%s' % unknown_content_['sha1'])) self.assertEqual(actual_ctags, []) @given(content()) def test_lookup_content_filetype(self, content): self.content_add_mimetype(content['sha1']) actual_filetype = service.lookup_content_filetype(content['sha1']) expected_filetype = self.content_get_mimetype(content['sha1']) self.assertEqual(actual_filetype, expected_filetype) @pytest.mark.xfail # Language indexer is disabled. @given(content()) def test_lookup_content_language(self, content): self.content_add_language(content['sha1']) actual_language = service.lookup_content_language(content['sha1']) expected_language = self.content_get_language(content['sha1']) self.assertEqual(actual_language, expected_language) @given(contents_with_ctags()) def test_lookup_expression(self, contents_with_ctags): per_page = 10 expected_ctags = [] for content_sha1 in contents_with_ctags['sha1s']: if len(expected_ctags) == per_page: break self.content_add_ctags(content_sha1) for ctag in self.content_get_ctags(content_sha1): if len(expected_ctags) == per_page: break if ctag['name'] == contents_with_ctags['symbol_name']: del ctag['id'] ctag['sha1'] = content_sha1 expected_ctags.append(ctag) actual_ctags = \ list(service.lookup_expression(contents_with_ctags['symbol_name'], last_sha1=None, per_page=10)) self.assertEqual(actual_ctags, expected_ctags) def test_lookup_expression_no_result(self): expected_ctags = [] actual_ctags = \ list(service.lookup_expression('barfoo', last_sha1=None, per_page=10)) self.assertEqual(actual_ctags, expected_ctags) @pytest.mark.skipif(fossology_missing, reason="requires fossology-nomossa installed") @given(content()) def test_lookup_content_license(self, content): self.content_add_license(content['sha1']) actual_license = service.lookup_content_license(content['sha1']) expected_license = self.content_get_license(content['sha1']) self.assertEqual(actual_license, expected_license) def test_stat_counters(self): actual_stats = service.stat_counters() self.assertEqual(actual_stats, self.storage.stat_counters()) @given(new_origin(), visit_dates()) def test_lookup_origin_visits(self, new_origin, visit_dates): self.storage.origin_add_one(new_origin) for ts in visit_dates: - self.storage.origin_visit_add(new_origin['url'], ts) + self.storage.origin_visit_add(new_origin['url'], ts, type='git') actual_origin_visits = list( service.lookup_origin_visits(new_origin['url'], per_page=100)) expected_visits = self.origin_visit_get(new_origin['url']) for expected_visit in expected_visits: expected_visit['origin'] = new_origin['url'] self.assertEqual(actual_origin_visits, expected_visits) @given(new_origin(), visit_dates()) def test_lookup_origin_visit(self, new_origin, visit_dates): self.storage.origin_add_one(new_origin) visits = [] for ts in visit_dates: visits.append(self.storage.origin_visit_add( - new_origin['url'], ts)) + new_origin['url'], ts, type='git')) visit = random.choice(visits)['visit'] actual_origin_visit = service.lookup_origin_visit( new_origin['url'], visit) expected_visit = dict(self.storage.origin_visit_get_by( new_origin['url'], visit)) expected_visit['date'] = expected_visit['date'].isoformat() expected_visit['metadata'] = {} expected_visit['origin'] = new_origin['url'] self.assertEqual(actual_origin_visit, expected_visit) @given(new_origin()) def test_lookup_origin(self, new_origin): self.storage.origin_add_one(new_origin) - actual_origin = service.lookup_origin({'type': new_origin['type'], - 'url': new_origin['url']}) - expected_origin = self.storage.origin_get({'type': new_origin['type'], - 'url': new_origin['url']}) + actual_origin = service.lookup_origin({'url': new_origin['url']}) + expected_origin = self.storage.origin_get({'url': new_origin['url']}) self.assertEqual(actual_origin, expected_origin) @given(invalid_sha1()) def test_lookup_release_ko_id_checksum_not_a_sha1(self, invalid_sha1): with self.assertRaises(BadInputExc) as cm: service.lookup_release(invalid_sha1) self.assertIn('invalid checksum', cm.exception.args[0].lower()) @given(sha256()) def test_lookup_release_ko_id_checksum_too_long(self, sha256): with self.assertRaises(BadInputExc) as cm: service.lookup_release(sha256) self.assertEqual('Only sha1_git is supported.', cm.exception.args[0]) @given(directory()) def test_lookup_directory_with_path_not_found(self, directory): path = 'some/invalid/path/here' with self.assertRaises(NotFoundExc) as cm: service.lookup_directory_with_path(directory, path) self.assertEqual('Directory entry with path %s from %s ' 'not found' % (path, directory), cm.exception.args[0]) @given(directory()) def test_lookup_directory_with_path_found(self, directory): directory_content = self.directory_ls(directory) directory_entry = random.choice(directory_content) path = directory_entry['name'] actual_result = service.lookup_directory_with_path(directory, path) self.assertEqual(actual_result, directory_entry) @given(release()) def test_lookup_release(self, release): actual_release = service.lookup_release(release) self.assertEqual(actual_release, self.release_get(release)) @given(revision(), invalid_sha1(), sha256()) def test_lookup_revision_with_context_ko_not_a_sha1(self, revision, invalid_sha1, sha256): sha1_git_root = revision sha1_git = invalid_sha1 with self.assertRaises(BadInputExc) as cm: service.lookup_revision_with_context(sha1_git_root, sha1_git) self.assertIn('Invalid checksum query string', cm.exception.args[0]) sha1_git = sha256 with self.assertRaises(BadInputExc) as cm: service.lookup_revision_with_context(sha1_git_root, sha1_git) self.assertIn('Only sha1_git is supported', cm.exception.args[0]) @given(revision(), unknown_revision()) def test_lookup_revision_with_context_ko_sha1_git_does_not_exist( self, revision, unknown_revision): sha1_git_root = revision sha1_git = unknown_revision with self.assertRaises(NotFoundExc) as cm: service.lookup_revision_with_context(sha1_git_root, sha1_git) self.assertIn('Revision %s not found' % sha1_git, cm.exception.args[0]) @given(revision(), unknown_revision()) def test_lookup_revision_with_context_ko_root_sha1_git_does_not_exist( self, revision, unknown_revision): sha1_git_root = unknown_revision sha1_git = revision with self.assertRaises(NotFoundExc) as cm: service.lookup_revision_with_context(sha1_git_root, sha1_git) self.assertIn('Revision root %s not found' % sha1_git_root, cm.exception.args[0]) @given(ancestor_revisions()) def test_lookup_revision_with_context(self, ancestor_revisions): sha1_git = ancestor_revisions['sha1_git'] root_sha1_git = ancestor_revisions['sha1_git_root'] for sha1_git_root in (root_sha1_git, {'id': hash_to_bytes(root_sha1_git)}): actual_revision = \ service.lookup_revision_with_context(sha1_git_root, sha1_git) children = [] for rev in self.revision_log(root_sha1_git): for p_rev in rev['parents']: p_rev_hex = hash_to_hex(p_rev) if p_rev_hex == sha1_git: children.append(rev['id']) expected_revision = self.revision_get(sha1_git) expected_revision['children'] = children self.assertEqual(actual_revision, expected_revision) @given(non_ancestor_revisions()) def test_lookup_revision_with_context_ko(self, non_ancestor_revisions): sha1_git = non_ancestor_revisions['sha1_git'] root_sha1_git = non_ancestor_revisions['sha1_git_root'] with self.assertRaises(NotFoundExc) as cm: service.lookup_revision_with_context(root_sha1_git, sha1_git) self.assertIn('Revision %s is not an ancestor of %s' % (sha1_git, root_sha1_git), cm.exception.args[0]) def test_lookup_directory_with_revision_not_found(self): unknown_revision_ = random_sha1() with self.assertRaises(NotFoundExc) as cm: service.lookup_directory_with_revision(unknown_revision_) self.assertIn('Revision %s not found' % unknown_revision_, cm.exception.args[0]) def test_lookup_directory_with_revision_unknown_content(self): unknown_content_ = random_content() unknown_revision_ = random_sha1() unknown_directory_ = random_sha1() dir_path = 'README.md' # Create a revision that points to a directory # Which points to unknown content revision = { 'author': { 'name': b'abcd', 'email': b'abcd@company.org', 'fullname': b'abcd abcd' }, 'committer': { 'email': b'aaaa@company.org', 'fullname': b'aaaa aaa', 'name': b'aaa' }, 'committer_date': { 'negative_utc': False, 'offset': 0, 'timestamp': 1437511651 }, 'date': { 'negative_utc': False, 'offset': 0, 'timestamp': 1437511651 }, 'message': b'bleh', 'metadata': [], 'parents': [], 'synthetic': False, 'type': 'git', 'id': hash_to_bytes(unknown_revision_), 'directory': hash_to_bytes(unknown_directory_) } # A directory that points to unknown content dir = { 'id': hash_to_bytes(unknown_directory_), 'entries': [{ 'name': bytes(dir_path.encode('utf-8')), 'type': 'file', 'target': hash_to_bytes(unknown_content_['sha1_git']), 'perms': DentryPerms.content }] } # Add the directory and revision in mem self.storage.directory_add([dir]) self.storage.revision_add([revision]) with self.assertRaises(NotFoundExc) as cm: service.lookup_directory_with_revision( unknown_revision_, dir_path) self.assertIn('Content not found for revision %s' % unknown_revision_, cm.exception.args[0]) @given(revision()) def test_lookup_directory_with_revision_ko_path_to_nowhere( self, revision): invalid_path = 'path/to/something/unknown' with self.assertRaises(NotFoundExc) as cm: service.lookup_directory_with_revision(revision, invalid_path) exception_text = cm.exception.args[0].lower() self.assertIn('directory or file', exception_text) self.assertIn(invalid_path, exception_text) self.assertIn('revision %s' % revision, exception_text) self.assertIn('not found', exception_text) @given(revision_with_submodules()) def test_lookup_directory_with_revision_submodules( self, revision_with_submodules): rev_sha1_git = revision_with_submodules['rev_sha1_git'] rev_dir_path = revision_with_submodules['rev_dir_rev_path'] actual_data = service.lookup_directory_with_revision( rev_sha1_git, rev_dir_path) revision = self.revision_get(revision_with_submodules['rev_sha1_git']) directory = self.directory_ls(revision['directory']) rev_entry = next(e for e in directory if e['name'] == rev_dir_path) expected_data = { 'content': self.revision_get(rev_entry['target']), 'path': rev_dir_path, 'revision': rev_sha1_git, 'type': 'rev' } self.assertEqual(actual_data, expected_data) @given(revision()) def test_lookup_directory_with_revision_without_path(self, revision): actual_directory_entries = \ service.lookup_directory_with_revision(revision) revision_data = self.revision_get(revision) expected_directory_entries = \ self.directory_ls(revision_data['directory']) self.assertEqual(actual_directory_entries['type'], 'dir') self.assertEqual(actual_directory_entries['content'], expected_directory_entries) @given(revision()) def test_lookup_directory_with_revision_with_path(self, revision): revision_data = self.revision_get(revision) dir_entries = [e for e in self.directory_ls(revision_data['directory']) if e['type'] in ('file', 'dir')] expected_dir_entry = random.choice(dir_entries) actual_dir_entry = \ service.lookup_directory_with_revision(revision, expected_dir_entry['name']) self.assertEqual(actual_dir_entry['type'], expected_dir_entry['type']) self.assertEqual(actual_dir_entry['revision'], revision) self.assertEqual(actual_dir_entry['path'], expected_dir_entry['name']) if actual_dir_entry['type'] == 'file': del actual_dir_entry['content']['checksums']['blake2s256'] for key in ('checksums', 'status', 'length'): self.assertEqual(actual_dir_entry['content'][key], expected_dir_entry[key]) else: sub_dir_entries = self.directory_ls(expected_dir_entry['target']) self.assertEqual(actual_dir_entry['content'], sub_dir_entries) @given(revision()) def test_lookup_directory_with_revision_with_path_to_file_and_data( self, revision): revision_data = self.revision_get(revision) dir_entries = [e for e in self.directory_ls(revision_data['directory']) if e['type'] == 'file'] expected_dir_entry = random.choice(dir_entries) expected_data = \ self.content_get(expected_dir_entry['checksums']['sha1']) actual_dir_entry = \ service.lookup_directory_with_revision(revision, expected_dir_entry['name'], with_data=True) self.assertEqual(actual_dir_entry['type'], expected_dir_entry['type']) self.assertEqual(actual_dir_entry['revision'], revision) self.assertEqual(actual_dir_entry['path'], expected_dir_entry['name']) del actual_dir_entry['content']['checksums']['blake2s256'] for key in ('checksums', 'status', 'length'): self.assertEqual(actual_dir_entry['content'][key], expected_dir_entry[key]) self.assertEqual(actual_dir_entry['content']['data'], expected_data['data']) @given(revision()) def test_lookup_revision(self, revision): actual_revision = service.lookup_revision(revision) self.assertEqual(actual_revision, self.revision_get(revision)) @given(new_revision()) def test_lookup_revision_invalid_msg(self, new_revision): new_revision['message'] = b'elegant fix for bug \xff' self.storage.revision_add([new_revision]) revision = service.lookup_revision(hash_to_hex(new_revision['id'])) self.assertEqual(revision['message'], None) self.assertEqual(revision['message_decoding_failed'], True) @given(new_revision()) def test_lookup_revision_msg_ok(self, new_revision): self.storage.revision_add([new_revision]) revision_message = service.lookup_revision_message( hash_to_hex(new_revision['id'])) self.assertEqual(revision_message, {'message': new_revision['message']}) def test_lookup_revision_msg_no_rev(self): unknown_revision_ = random_sha1() with self.assertRaises(NotFoundExc) as cm: service.lookup_revision_message(unknown_revision_) self.assertEqual( cm.exception.args[0], 'Revision with sha1_git %s not found.' % unknown_revision_ ) @given(revisions()) def test_lookup_revision_multiple(self, revisions): actual_revisions = list(service.lookup_revision_multiple(revisions)) expected_revisions = [] for rev in revisions: expected_revisions.append(self.revision_get(rev)) self.assertEqual(actual_revisions, expected_revisions) def test_lookup_revision_multiple_none_found(self): unknown_revisions_ = [random_sha1(), random_sha1(), random_sha1()] actual_revisions = \ list(service.lookup_revision_multiple(unknown_revisions_)) self.assertEqual(actual_revisions, [None] * len(unknown_revisions_)) @given(revision()) def test_lookup_revision_log(self, revision): actual_revision_log = \ list(service.lookup_revision_log(revision, limit=25)) expected_revision_log = self.revision_log(revision, limit=25) self.assertEqual(actual_revision_log, expected_revision_log) def _get_origin_branches(self, origin): origin_visit = self.origin_visit_get(origin['url'])[-1] snapshot = self.snapshot_get(origin_visit['snapshot']) branches = {k: v for (k, v) in snapshot['branches'].items() if v['target_type'] == 'revision'} return branches @given(origin()) def test_lookup_revision_log_by(self, origin): branches = self._get_origin_branches(origin) branch_name = random.choice(list(branches.keys())) actual_log = \ list(service.lookup_revision_log_by(origin['url'], branch_name, None, limit=25)) expected_log = \ self.revision_log(branches[branch_name]['target'], limit=25) self.assertEqual(actual_log, expected_log) @given(origin()) def test_lookup_revision_log_by_notfound(self, origin): with self.assertRaises(NotFoundExc): service.lookup_revision_log_by( origin['url'], 'unknown_branch_name', None, limit=100) def test_lookup_content_raw_not_found(self): unknown_content_ = random_content() with self.assertRaises(NotFoundExc) as cm: service.lookup_content_raw('sha1:' + unknown_content_['sha1']) self.assertIn(cm.exception.args[0], 'Content with %s checksum equals to %s not found!' % ('sha1', unknown_content_['sha1'])) @given(content()) def test_lookup_content_raw(self, content): actual_content = service.lookup_content_raw( 'sha256:%s' % content['sha256']) expected_content = self.content_get(content['sha1']) self.assertEqual(actual_content, expected_content) def test_lookup_content_not_found(self): unknown_content_ = random_content() with self.assertRaises(NotFoundExc) as cm: service.lookup_content('sha1:%s' % unknown_content_['sha1']) self.assertIn(cm.exception.args[0], 'Content with %s checksum equals to %s not found!' % ('sha1', unknown_content_['sha1'])) @given(content()) def test_lookup_content_with_sha1(self, content): actual_content = service.lookup_content( 'sha1:%s' % content['sha1']) expected_content = self.content_get_metadata(content['sha1']) self.assertEqual(actual_content, expected_content) @given(content()) def test_lookup_content_with_sha256(self, content): actual_content = service.lookup_content( 'sha256:%s' % content['sha256']) expected_content = self.content_get_metadata(content['sha1']) self.assertEqual(actual_content, expected_content) def test_lookup_directory_bad_checksum(self): with self.assertRaises(BadInputExc): service.lookup_directory('directory_id') def test_lookup_directory_not_found(self): unknown_directory_ = random_sha1() with self.assertRaises(NotFoundExc) as cm: service.lookup_directory(unknown_directory_) self.assertIn('Directory with sha1_git %s not found' % unknown_directory_, cm.exception.args[0]) @given(directory()) def test_lookup_directory(self, directory): actual_directory_ls = list(service.lookup_directory( directory)) expected_directory_ls = self.directory_ls(directory) self.assertEqual(actual_directory_ls, expected_directory_ls) @given(empty_directory()) def test_lookup_directory_empty(self, empty_directory): actual_directory_ls = list(service.lookup_directory(empty_directory)) self.assertEqual(actual_directory_ls, []) @given(origin()) def test_lookup_revision_by_nothing_found(self, origin): with self.assertRaises(NotFoundExc): service.lookup_revision_by( origin['url'], 'invalid-branch-name') @given(origin()) def test_lookup_revision_by(self, origin): branches = self._get_origin_branches(origin) branch_name = random.choice(list(branches.keys())) actual_revision = \ service.lookup_revision_by(origin['url'], branch_name, None) expected_revision = \ self.revision_get(branches[branch_name]['target']) self.assertEqual(actual_revision, expected_revision) @given(origin(), revision()) def test_lookup_revision_with_context_by_ko(self, origin, revision): with self.assertRaises(NotFoundExc): service.lookup_revision_with_context_by(origin['url'], 'invalid-branch-name', None, revision) @given(origin()) def test_lookup_revision_with_context_by(self, origin): branches = self._get_origin_branches(origin) branch_name = random.choice(list(branches.keys())) root_rev = branches[branch_name]['target'] root_rev_log = self.revision_log(root_rev) children = defaultdict(list) for rev in root_rev_log: for rev_p in rev['parents']: children[rev_p].append(rev['id']) rev = root_rev_log[-1]['id'] actual_root_rev, actual_rev = service.lookup_revision_with_context_by( origin['url'], branch_name, None, rev) expected_root_rev = self.revision_get(root_rev) expected_rev = self.revision_get(rev) expected_rev['children'] = children[rev] self.assertEqual(actual_root_rev, expected_root_rev) self.assertEqual(actual_rev, expected_rev) def test_lookup_revision_through_ko_not_implemented(self): with self.assertRaises(NotImplementedError): service.lookup_revision_through({ 'something-unknown': 10, }) @given(origin()) def test_lookup_revision_through_with_context_by(self, origin): branches = self._get_origin_branches(origin) branch_name = random.choice(list(branches.keys())) root_rev = branches[branch_name]['target'] root_rev_log = self.revision_log(root_rev) rev = root_rev_log[-1]['id'] self.assertEqual(service.lookup_revision_through({ 'origin_url': origin['url'], 'branch_name': branch_name, 'ts': None, 'sha1_git': rev }), service.lookup_revision_with_context_by( origin['url'], branch_name, None, rev) ) @given(origin()) def test_lookup_revision_through_with_revision_by(self, origin): branches = self._get_origin_branches(origin) branch_name = random.choice(list(branches.keys())) self.assertEqual(service.lookup_revision_through({ 'origin_url': origin['url'], 'branch_name': branch_name, 'ts': None, }), service.lookup_revision_by( origin['url'], branch_name, None) ) @given(ancestor_revisions()) def test_lookup_revision_through_with_context(self, ancestor_revisions): sha1_git = ancestor_revisions['sha1_git'] sha1_git_root = ancestor_revisions['sha1_git_root'] self.assertEqual(service.lookup_revision_through({ 'sha1_git_root': sha1_git_root, 'sha1_git': sha1_git, }), service.lookup_revision_with_context( sha1_git_root, sha1_git) ) @given(revision()) def test_lookup_revision_through_with_revision(self, revision): self.assertEqual(service.lookup_revision_through({ 'sha1_git': revision }), service.lookup_revision(revision) ) @given(revision()) def test_lookup_directory_through_revision_ko_not_found(self, revision): with self.assertRaises(NotFoundExc): service.lookup_directory_through_revision( {'sha1_git': revision}, 'some/invalid/path') @given(revision()) def test_lookup_directory_through_revision_ok(self, revision): revision_data = self.revision_get(revision) dir_entries = [e for e in self.directory_ls(revision_data['directory']) if e['type'] == 'file'] dir_entry = random.choice(dir_entries) self.assertEqual( service.lookup_directory_through_revision({'sha1_git': revision}, dir_entry['name']), (revision, service.lookup_directory_with_revision( revision, dir_entry['name'])) ) @given(revision()) def test_lookup_directory_through_revision_ok_with_data(self, revision): revision_data = self.revision_get(revision) dir_entries = [e for e in self.directory_ls(revision_data['directory']) if e['type'] == 'file'] dir_entry = random.choice(dir_entries) self.assertEqual( service.lookup_directory_through_revision({'sha1_git': revision}, dir_entry['name'], with_data=True), (revision, service.lookup_directory_with_revision( revision, dir_entry['name'], with_data=True)) ) diff --git a/swh/web/tests/misc/test_origin_save.py b/swh/web/tests/misc/test_origin_save.py index 2cf41bcd..a6fd62ae 100644 --- a/swh/web/tests/misc/test_origin_save.py +++ b/swh/web/tests/misc/test_origin_save.py @@ -1,86 +1,86 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime from unittest.mock import patch from rest_framework.test import APITestCase, APIClient from swh.web.common.origin_save import ( SAVE_REQUEST_ACCEPTED, SAVE_TASK_NOT_YET_SCHEDULED ) from swh.web.common.utils import reverse from swh.web.settings.tests import save_origin_rate_post from swh.web.tests.testcase import WebTestCase class SwhOriginSaveTest(WebTestCase, APITestCase): def setUp(self): self.client = APIClient(enforce_csrf_checks=True) + self.visit_type = 'git' self.origin = { - 'type': 'git', 'url': 'https://github.com/python/cpython' } @patch('swh.web.misc.origin_save.create_save_origin_request') def test_save_request_form_csrf_token( self, mock_create_save_origin_request): self._mock_create_save_origin_request(mock_create_save_origin_request) url = reverse('origin-save-request', - url_args={'origin_type': self.origin['type'], + url_args={'visit_type': self.visit_type, 'origin_url': self.origin['url']}) resp = self.client.post(url) self.assertEqual(resp.status_code, 403) data = self._get_csrf_token(reverse('origin-save')) resp = self.client.post(url, data=data) self.assertEqual(resp.status_code, 200) @patch('swh.web.misc.origin_save.create_save_origin_request') def test_save_request_form_rate_limit( self, mock_create_save_origin_request): self._mock_create_save_origin_request(mock_create_save_origin_request) url = reverse('origin-save-request', - url_args={'origin_type': self.origin['type'], + url_args={'visit_type': self.visit_type, 'origin_url': self.origin['url']}) data = self._get_csrf_token(reverse('origin-save')) for _ in range(save_origin_rate_post): resp = self.client.post(url, data=data) self.assertEqual(resp.status_code, 200) resp = self.client.post(url, data=data) self.assertEqual(resp.status_code, 429) def test_old_save_url_redirection(self): url = reverse('browse-origin-save') resp = self.client.get(url) self.assertEqual(resp.status_code, 302) redirect_url = reverse('origin-save') self.assertEqual(resp['location'], redirect_url) def _get_csrf_token(self, url): resp = self.client.get(url) return { 'csrfmiddlewaretoken': resp.cookies['csrftoken'].value } def _mock_create_save_origin_request(self, mock): expected_data = { - 'origin_type': self.origin['type'], + 'visit_type': self.visit_type, 'origin_url': self.origin['url'], 'save_request_date': datetime.now().isoformat(), 'save_request_status': SAVE_REQUEST_ACCEPTED, 'save_task_status': SAVE_TASK_NOT_YET_SCHEDULED, 'visit_date': None } mock.return_value = expected_data