diff --git a/README b/README index bfe45627..8d5880d7 100644 --- a/README +++ b/README @@ -1,4 +1,16 @@ -swh-web-api -=========== +swh-web +======= -SWH's API + web front-end +This repository holds the developement of Software Heritage web applications: + + * SWH Web API (https://archive.softwareheritage.org/api): enables to query + the content of the SWH archive through HTTP requests and get responses in JSON or YAML. + + * SWH Web browse (https://archive.softwareheritage.org/browse): graphical interface + that eases the navigation in the SWH archive. + +Those applications are powered by the [Django Web Framework](https://www.djangoproject.com/). + +Documentation about how to use these components but also the details of their URI schemes +can be found in the docs folder. The produced HTML documentation can be read and browsed +at https://docs.softwareheritage.org/devel/swh-web/index.html. diff --git a/README-dev.md b/README-dev.md deleted file mode 100644 index 2304f4cc..00000000 --- a/README-dev.md +++ /dev/null @@ -1,76 +0,0 @@ -Developers Information -====================== - -Run server ----------- - -Either use the django manage script directly (useful in development mode as it offers various commands). -The configuration will be taken from the default configuration file: '~/.config/swh/webapp/webapp.yml'. - -``` -python3 -m swh.web.manage runserver -``` - -or use the following shortcut: - -``` -make run -``` - -Sample configuration --------------------- - -The following introduces a default configuration file: -``` -storage: - cls: remote - args: - url: http://localhost:5002 -debug: false -throttling: - cache_uri: None - scopes: - swh_api: - limiter_rate: 120/h - exempted_networks: - - 127.0.0.0/8 -``` - -modules' description --------------------- - -### Layers - -Folder swh/web/api/: - -- views main api endpoints definitions (html browsable + json + yaml) -- service Orchestration layer used by views module. - In charge of communication with swh storage to retrieve - information and conversion for the upper layer. - -In short: -1. views -depends-> service ----asks----> swh-storage -2. views <- service <----rets---- swh-storage - -### Utilities - -Folder swh/web/api/: - -- apidoc Browsable api functions. -- apiresponse Api response utility functions -- apiurls Api routes registration functions -- exc Exception definitions -- converters conversion layer to transform swh data to serializable data. - Used by `service` to convert data before transmitting to `api` or `views`. -- query Utilities to parse data from http endpoints. - Used by `service` -- utils Utilities used throughout swh-web-api. - -### About apidoc - -This is a 'decorator tower' that stores the data associated with the -documentation upon loading the apidoc module. The top decorator of any -tower should be @apidoc.route(). Apidoc raises an exception if this -decorator is missing, and flask raises an exception if it is present -but not at the top of the tower. - diff --git a/README-uri-scheme.md b/README-uri-scheme.md deleted file mode 100644 index d820f772..00000000 --- a/README-uri-scheme.md +++ /dev/null @@ -1,504 +0,0 @@ -URI scheme -========== - - -User URLs ---------- - -### Context-independent browsing - -Context-independent URLs provide information about SWH objects (e.g., -revisions, directories, contents, person, ...), independently of the -contexts where they have been found (e.g., specific repositories, -branches, commits, ...). - -The following endpoints are the same of the API case (see below), and -just render the corresponding information for user consumption. Where -hyperlinks are created, they always point to other context-independent -user URLs: - -* /content/[:]/ Information on content -* /content/[:]/raw/ Display the content data -* /content/[:]/origin/ Display information on content with its origin information (Deactivated) -* /directory// Browse directory's files -* /origin// Information on origin -* /person// Information on person -* /release// Information on release -* /entity// Information on Entity with hierarchy -* /revision// Browse revision -* /revision//log/ Revision log from - -Currently, the above endpoints are mounted below the top-level /browse/ -namespace. - - -### Context-dependent browsing - -Context-dependent URLs provide information about SWH objects, limited to -specific contexts where the objects have been found. For example, users might -want to see: - -- the commits that descend (i.e., are based on and hence more recent) from a - given commit but only in a given repository, ignoring "forks" elsewhere - -- the parent directory of a given one, limited to a specific revision - and starting root directory (note indeed that in the general case a - given directory might be mounted in multiple places, which might - vary across revisions) - - -### Context: a specific revision (AKA commit) - -* /revision// - - Show information about a given revision, pointing to parent - revisions only (i.e., no links/info about child revisions as they - cannot be limited a priori). Links to parent revisions maintains a - reference to , using the /history/ URL scheme (see below). - -* /revision//root// - - Show information about revision , limited to the sub-graph - rooted at . The obtained page show both parent and child - revisions of , but exclude all revisions that are *not* - transitively reachable (going back in time) from . - - Links to all revisions SHA1_GIT' reachable from are of the - form /revision//root/, where - remains unchanged. In the degenerate case of browsing back to the root - revision, we might end up on the URL - /revision//root// where SHA1_GIT_1 == SHA1_GIT_2. - That URL is equivalent to /revision// and might be simplified - redirecting to it. - - **Workaround**. Currently, we cannot quickly check whether SHA1_GIT_CUR is - reachable from SHA1_GIT_ROOT. Therefore we adopt the following (sub-optimal - and incomplete) endpoint instead. - - * /revision//prev/,[...], - - where ,[...], is a path in the revision graph - leading to , i.e., SHA1_GIT_N is a revision pointing directly - to SHA1_GIT_CUR, SHA1_GIT_(N-1) points directly to SHA1_GIT_N, etc. - - They path might be empty, complete w.r.t. the user navigation history up to - SHA1_GIT_CUR, or incomplete. The UI will show (some of) the most near - revisions in the path as previous commits and will allow users to jump to - them. When following parent revisions of (going back in - time), the path is extended, possibly trimming it to a maximum size; when - following descendant revisions of , choosing from path - elements, the path is trimmed to the selected revision. - - Note: it is possible for users to "cheat" and create URLs where the given - revision path does not match the reality of our revision graph. Well, too - bad for them. They will get pages whose navigation breadcrumbs do not - reflect reality. - -* /revision//directory/[] -* /revision//root//directory/[] - - Starting from the revision identified as in the previous URLs, navigate the - directory associated to that revision. - - When is absent, show the content of the root directory for the given - revision. When is present, treat it as a local path starting at that - root directory, resolve it, and show the content of the obtained directory. - - Links to *sub*-directory/files append new parts to . Links to parent - directories remove trailing parts of . Note that this latter operation - is well-defined, given that we are looking at a specific revision and - navigation starts at the root directory. - - -### Context: a specific point in spacetime - -Instead of having to specify a (root) revision by SHA1_GIT, users might want to -specify a place and a time. In SWH a "place" is an origin, with an optional -branch name; a "time" is a timestamp at which some place has been observed by -SWH crawlers. - -Wherever a revision context is expected in a path (i.e., a -"/revision//" path fragment) we can put in its stead a path fragment -of the form /origin/[/branch/][/ts//]. Such a -fragment is resolved, internally by the SWH archive, to a SHA1_GIT as follows: - -- [if is absent] look for the most recent crawl of origin -- [if is given] look for the most recent crawl of origin - whose timestamp is <= -- [if is given] look for the branch -- [if is absent] look for branch "master" -- return the pointed by the chosen branch - -The already mentioned URLs for revision contexts can therefore be alternatively -specified by users as: - -* /revision/origin/[/branch/][/ts/]/ -* /revision/origin/[/branch/][/ts/]/history// -* /revision/origin/[/branch/][/ts/]/directory/[] -* /revision/origin/[/branch/][/ts/]/history//directory/[] - -Typing: - -- s are given as integer identifiers, pointing into the origin table. - There will be separate mechanisms for finding origins by other means (e.g., - URLs, metadata, etc). Once an origin is found, it can be used by ID into the - above URL schemes - -- names are given as per the corresponding VCS (e.g., Git) and might - therefore contains characters that are either invalid in URLs, or that might - make the above URL schemes ambiguous (e.g., '/'). All those characters will - need to be URL-escaped. (e.g., '/' will become '%2F') - -- s are given in a format as liberal as possible, to uphold the - principle of least surprise. At the very minimum it should be possible to - enter timestamps as: - - - ISO 8601 timestamps (see for instance the output of `date -I`, `date -Is`) - - YYYY[MM[DD[HH[MM[SS]]]]] ad-hoc format - - Implementation proposal: use Python dateutil's parser and be done with it - https://dateutil.readthedocs.org/en/latest/parser.html . Note: that dateutil - does *not* allow to use classical UNIX timestamps expressed as seconds since - the epoch (i.e., `date +%s` output). We will need to single case them. - - The same escaping considerations given for apply. - -Notes: - -- Differently from , s are still specified as SHA1 and - cannot be specified a origin/branch/ts triples. This is to preserve some URL - sanity. - - -API URLs --------- - -### Endpoints - -The api /api/1 is partially browsable on defined endpoints (/api, /api/1). - -* /api/ and /api/1/ - -List endpoint methods as per the client's 'Accept' header request. - -The following routes are to be anchored at at /api/1 - -* /revision/: show commit information - - $ curl -H 'Accept: application/json' http://localhost:6543/api/1/revision/18d8be353ed3480476f032475e7c233eff7371d5 - { - "author_email": "robot@softwareheritage.org", - "author_name": "Software Heritage", - "committer_date": "Mon, 17 Jan 2000 10:23:54 GMT", - "committer_date_offset": 0, - "committer_email": "robot@softwareheritage.org", - "committer_name": "Software Heritage", - "date": "Mon, 17 Jan 2000 10:23:54 GMT", - "date_offset": 0, - "directory": "7834ef7e7c357ce2af928115c6c6a42b7e2a44e6", - "id": "18d8be353ed3480476f032475e7c233eff7371d5", - "message": "synthetic revision message", - "metadata": { - "original_artifact": [ - { - "archive_type": "tar", - "name": "webbase-5.7.0.tar.gz", - "sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd", - "sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1", - "sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f309d36484e7edf7bb912" - } - ] - }, - "parents": [ - null - ], - "synthetic": true, - "type": "tar" - } - -* /directory/: show directory information (including ls) - - curl -X GET http://localhost:6543/api/1/directory/3126f46e2f7dc752227131a2a658265e58f53e38 - [ - { - "dir_id": "3126f46e2f7dc752227131a2a658265e58f53e38", - "name": "Makefile.am", - "perms": 100644, - "sha1": "b0283d8126f975e7b4a4348d13b07ddebe2cf8bf", - "sha1_git": "e0522786777256d57c5210219bcbe8dacdad273d", - "sha256": "897f3189dcfba96281b2190325c54afc74a42e2419c053baadfadc14386935ee", - "status": "visible", - "target": "e0522786777256d57c5210219bcbe8dacdad273d", - "type": "file" - }, - { - "dir_id": "3126f46e2f7dc752227131a2a658265e58f53e38", - "name": "Makefile.in", - "perms": 100644, - "sha1": "81f5757b9451811cfb3ef84612e45a973c70b4e6", - "sha1_git": "3b948d966fd8e99f93670025f63a550168d57d71", - "sha256": "f5acd84a40f05d997a36b8846c4872a92ee57083abb77c82e05e9763c8edb59a", - "status": "visible", - "target": "3b948d966fd8e99f93670025f63a550168d57d71", - "type": "file" - }, - - ... snip ... - - { - "dir_id": "3126f46e2f7dc752227131a2a658265e58f53e38", - "name": "webtools.h", - "perms": 100644, - "sha1": "4b4c942ddd490ec1e312074ddfac352097886c02", - "sha1_git": "e6fb8969d00e23dd152df5e7fb167118eab67342", - "sha256": "95ffe6c0108f6ec48ccb0c93e966b54f1494f5cc353b066644c11fa47766620f", - "status": "visible", - "target": "e6fb8969d00e23dd152df5e7fb167118eab67342", - "type": "file" - }, - { - "dir_id": "3126f46e2f7dc752227131a2a658265e58f53e38", - "name": "ylwrap", - "perms": 100644, - "sha1": "9073938df9ae47d585bfdf176bfff45d06f3e13e", - "sha1_git": "13fc38d75f2a47bc55e90ad5bf8d8a0184b14878", - "sha256": "184eb644e51154c79b42df70c22955b818d057491f84ca0e579e4f9e48a60d7b", - "status": "visible", - "target": "13fc38d75f2a47bc55e90ad5bf8d8a0184b14878", - "type": "file" - } - ] - -* /content/[:]: show content information - - - content is specified by HASH, according to HASH_ALGO, where HASH_ALGO is - one of: sha1, sha1_git, sha256. This means that several different URLs (at - least one per HASH_ALGO) will point to the same content - - HASH_ALGO defaults to "sha1" (?) - - curl -X GET http://localhost:6543/api/1/content/sha1:486b486d2a4998929c68265fa85ab2326db5528a - - { - "data": "/api/1/content/486b486d2a4998929c68265fa85ab2326db5528a/raw", - "sha1": "486b486d2a4998929c68265fa85ab2326db5528a" - } - - curl -X GET http://localhost:6543/api/1/content/sha1:4a1b6d7dd0a923ed90156c4e2f5db030095d8e08/ - - {"error": "Content with sha1:4a1b6d7dd0a923ed90156c4e2f5db030095d8e08 not found."} - -* /content/[/raw - - curl -H 'Accept: text/plain' http://localhost:6543/api/1/content/sha1:486b486d2a4998929c68265fa85ab2326db5528a/raw - - The GNU cfs-el web homepage is at - @uref{http://www.gnu.org/software/cfs-el/cfs-el.html}. - - You can find the latest distribution of GNU cfs-el at - @uref{ftp://ftp.gnu.org/gnu/} or at any of its mirrors. - - -* /release/: show release information - -Sample: - - $ curl -X GET http://localhost:6543/api/1/release/4a1b6d7dd0a923ed90156c4e2f5db030095d8e08 - { - "author_name": "Software Heritage", - "author_email": "robot@softwareheritage.org", - "comment": "synthetic release message", - "date": "Sat, 04 Mar 2000 07:50:35 GMT", - "date_offset": 0, - "id": "4a1b6d7dd0a923ed90156c4e2f5db030095d8e08", - "name": "4.0.6", - "revision": "5c7814ce9978d4e16f3858925b5cea611e500eec", - "synthetic": true - }% - -* /person/: show person information - - curl http://localhost:6543/api/1/person/1 - { - "email": "robot@softwareheritage.org", - "id": 1, - "name": "Software Heritage" - } - - curl http://localhost:6543/api/1/person/2 - {"error": "Person with id 2 not found."} - -* /origin/: show origin information - -Sample: - - $ curl -X GET http://localhost:6543/api/1/origin/1 - { - "id": 1, - "lister": null, - "project": null, - "type": "ftp", - "url": "rsync://ftp.gnu.org/old-gnu/solfege" - }% - -* /browse/ - - TODO: rename this to something more explicit about the fact we want more - information about some content - -Return content information up to one of its origin if the content is -found. - - curl http://localhost:6543/api/1/browse/sha1:2e98ab73456aad8dfc6cc50d562ee1b80d201753 - { - "path": "republique.py", - "origin_url": "file:///dev/null", - "origin_type": "git", - "revision": "8f8640a1c024c2ef85fa8e8d9297ea289134472d", - "branch": "refs/remotes/origin/master" - } - -* /uploadnsearch/ - - TODO: remove this? - -Post a file's content to api. -Api computes the sha1 hash and checks in the storage if such sha1 exists. -Json answer: - - {'sha1': hexadecimal sha1, - 'found': true or false} - -Sample: - - $ curl -X POST -F filename=@/path/to/file http://localhost:6543/api/1/uploadnsearch - { - "found": false, - "sha1": "e95097ad2d607b4c89c1ce7ca1fef2a1e4450558" - }% - -* /revision//log - -Show all revisions (~git log) starting from . -The first element is the given sha1_git. - -Sample: - - curl http://localhost:6543/api/1/revision/7026b7c1a2af56521e951c01ed20f255fa054238/log/ - - [ - { - "id": "7026b7c1a2af56521e951c01ed20f255fa054238", - "parents": [], - "type": "git", - "committer_date": "Mon, 12 Oct 2015 11:05:53 GMT", - "synthetic": false, - "committer": { - "email": "a3nm@a3nm.net", - "name": "Antoine Amarilli" - }, - "message": "+1 limitation\n", - "author": { - "email": "a3nm@a3nm.net", - "name": "Antoine Amarilli" - }, - "date": "Mon, 12 Oct 2015 11:05:53 GMT", - "metadata": null, - "directory": "a33a9acf2419b9a291e8a02302e6347dcffde5a6" - }, - { - "id": "368a48fe15b7db2383775f97c6b247011b3f14f4", - "parents": [], - "type": "git", - "committer_date": "Mon, 12 Oct 2015 10:57:11 GMT", - "synthetic": false, - "committer": { - "email": "a3nm@a3nm.net", - "name": "Antoine Amarilli" - }, - "message": "actually fix bug\n", - "author": { - "email": "a3nm@a3nm.net", - "name": "Antoine Amarilli" - }, - "date": "Mon, 12 Oct 2015 10:57:11 GMT", - "metadata": null, - "directory": "1d5188e4991510c74d62272f0301352c5c1b850b" - }, - ... - ] - -* /project/: show project information - -* /organization/: show organization information - -* /directory//path/to/file-or-dir: ditto, but for file or directory pointed by path - - - note: This is the same as /directory/, where - is the sha1_git ID of the directory pointed by path or - /content/sha1_git: (for content) - - -### Global behavior - -The api routes outputs 'application/json' as default. - -#### Accept header - -Also, you can specify the following 'Accept' header in your client query: -- application/json -- application/yaml -- text/html - -The client can use specific filters and compose them as (s)he sees fit. - -#### Fields - -The client can filter the result output by field names when requesting -`application/json` or `application/yaml` output. - -Ex: - - curl http://localhost:6543/api/1/stat/counters?fields=revision,release,content - { - "content": 133616, - "revision": 1042, - "release": 660 - } - -#### JSONP - -When using the accept header 'application/json', the route can be -enhanced by adding a `callback` parameter. This will output the -result in a json function whose name is the callback parameter - -Ex: - - curl http://localhost:6543/api/1/stat/counters?callback=jsonp&fields=directory_entry_dir,revision,entity - - jsonp({ - "directory_entry_dir": 12478, - "revision": 1042, - "entity": 0 - }) - -#### Error - -When an error is raised, the error code response is used: -- 400: user's input is not correct regarding the API -- 404: user's input is ok but we did not found what (s)he was looking for -- 503: temporary internal server error (backend is down for example) - -And the body of the response should be a dictionary with some more information on the error. - -Bad request sample: - - curl http://localhost:6543/api/1/revision/18d8be353ed3480476f032475e7c233eff7371d - {"error": "Invalid checksum query string 18d8be353ed3480476f032475e7c233eff7371d"} - - curl http://localhost:6543/api/1/revision/sha1:18d8be353ed3480476f032475e7c233eff7371d - {"error": "Invalid hash 18d8be353ed3480476f032475e7c233eff7371d for algorithm sha1"} - -Not found sample: - - curl http://localhost:6543/api/1/revision/sha1:18d8be353ed3480476f032475e7c233eff7371df - {"error": "Revision with sha1_git sha1:18d8be353ed3480476f032475e7c233eff7371df not found."} diff --git a/docs/Makefile b/docs/Makefile index e76f15d8..7a28117f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,7 +1 @@ -include ../../swh-docs/Makefile.sphinx - -html: copy_md - -copy_md: - cp ../README-dev.md dev-info.md - cp ../README-uri-scheme.md uri-scheme.md +include ../../swh-docs/Makefile.sphinx \ No newline at end of file diff --git a/docs/developers-info.rst b/docs/developers-info.rst new file mode 100644 index 00000000..e89a6b12 --- /dev/null +++ b/docs/developers-info.rst @@ -0,0 +1,128 @@ +Developers Information +====================== + +Sample configuration +-------------------- + +The configuration will be taken from the default configuration file: *~/.config/swh/webapp/webapp.yml*. +The following introduces a default configuration file: + +.. sourcecode:: yaml + + storage: + cls: remote + args: + url: http://localhost:5002 + debug: false + throttling: + cache_uri: None + scopes: + swh_api: + limiter_rate: 120/h + exempted_networks: + - 127.0.0.0/8 + +Run server +---------- + +Either use the django manage script directly (useful in development mode as it offers various commands): + +.. sourcecode:: shell + + $ python3 -m swh.web.manage runserver + +or use the following shortcut: + +.. sourcecode:: shell + + $ make run + +Modules description +------------------- + +Common to all web applications +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configuration and settings +"""""""""""""""""""""""""" + + * :mod:`swh.web.config`: holds the SWH configuration for the web applications. + * :mod:`swh.web.doc_config`: utility module used to extend the sphinx configuration + when building the documentation. + * :mod:`swh.web.manage`: Django management module for developpers. + * :mod:`swh.web.urls`: module that holds the whole URI scheme of all + the web applications. + * :mod:`swh.web.wsgi`: WSGI module to use when deploying the web applications + in production. + * :mod:`swh.web.settings.common`: Common Django settings + * :mod:`swh.web.settings.development`: Django settings for development + * :mod:`swh.web.settings.production`: Django settings for production + * :mod:`swh.web.settings.tests`: Django settings for tests + +Common utilities +"""""""""""""""" + + * :mod:`swh.web.common.converters`: conversion module used to transform SWH raw data + to serializable ones. It is used by :mod:`swh.web.common.service`: to convert data + before transmitting then to Django views. + * :mod:`swh.web.common.exc`: module defining exceptions used in the web applications. + * :mod:`swh.web.common.highlightjs`: utility module to ease the use of the highlightjs_ + library in produced Django views. + * :mod:`swh.web.common.query`: Utilities to parse data from HTTP endpoints. It is used + by :mod:`swh.web.common.service`. + * :mod:`swh.web.common.service`: Orchestration layer used by views module + in charge of communication with :mod:`swh.storage` to retrieve information and + perform conversion for the upper layer. + * :mod:`swh.web.common.swh_templatetags`: Custom Django template tags library for swh. + * :mod:`swh.web.common.throttling`: Custom request rate limiter to use with the `Django REST Framework + `_ + * :mod:`swh.web.common.urlsindex`: Utilities to help the registering of endpoints + for the web applications + * :mod:`swh.web.common.utils`: Utility functions used in the web applications implementation + + +SWH Web API application +^^^^^^^^^^^^^^^^^^^^^^^ + + * :mod:`swh.web.api.apidoc`: Utilities to document the web api for its html + browsable rendering. + * :mod:`swh.web.api.apiresponse`: Utility module to ease the generation of + web api responses. + * :mod:`swh.web.api.apiurls`: Utilities to facilitate the registration of SWH + web api endpoints. + * :mod:`swh.web.api.urls`: Module that defines the whole URI scheme for the api endpoints + * :mod:`swh.web.api.utils`: Utility functions used in the SWH web api implementation. + * :mod:`swh.web.api.views.content`: Implementation of API endpoints for getting information + about SWH contents. + * :mod:`swh.web.api.views.directory`: Implementation of API endpoints for getting information + about SWH directories. + * :mod:`swh.web.api.views.entity`: Implementation of API endpoints for getting information + about SWH entities. + * :mod:`swh.web.api.views.origin`: Implementation of API endpoints for getting information + about SWH origins. + * :mod:`swh.web.api.views.person`: Implementation of API endpoints for getting information + about SWH persons. + * :mod:`swh.web.api.views.release`: Implementation of API endpoints for getting information + about SWH releases. + * :mod:`swh.web.api.views.revision`: Implementation of API endpoints for getting information + about SWH revisions. + * :mod:`swh.web.api.views.stat`: Implementation of API endpoints for getting information + about SWH archive statistics. + * :mod:`swh.web.api.views.utils`: Utilities used in the web api endpoints implementation. + +SWH Web browse application +^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * :mod:`swh.web.browse.browseurls`: Utilities to facilitate the registration of SWH web + browse endpoints. + * :mod:`swh.web.browse.urls`: Module that defines the whole URI scheme for the SWH web + browse endpoints. + * :mod:`swh.web.browse.utils`: Utilities functions used troughout the SWH web browse + endpoints implementation. + * :mod:`swh.web.browse.views.content`: Implementation of endpoints for browsing SWH contents. + * :mod:`swh.web.browse.views.directory`: Implementation of endpoints for browsing SWH directories. + * :mod:`swh.web.browse.views.origin`: Implementation of endpoints for browsing SWH origins. + * :mod:`swh.web.browse.views.person`: Implementation of endpoints for browsing SWH persons. + * :mod:`swh.web.browse.views.revision`: Implementation of endpoints for browsing SWH revisions. + +.. _highlightjs: https://highlightjs.org/ diff --git a/docs/index.rst b/docs/index.rst index f747214f..dfebdba9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,19 +1,19 @@ Software Heritage Web Applications - Development Documentation =============================================================== .. toctree:: :maxdepth: 3 :caption: Contents: - dev-info.md + developers-info uri-scheme-api uri-scheme-browse Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` diff --git a/swh/web/api/views/content.py b/swh/web/api/views/content.py index 3ffa50fb..989db1d5 100644 --- a/swh/web/api/views/content.py +++ b/swh/web/api/views/content.py @@ -1,340 +1,340 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools from django.http import HttpResponse from swh.web.common import service from swh.web.common.utils import reverse from swh.web.common.exc import NotFoundExc, ForbiddenExc from swh.web.api import apidoc as api_doc from swh.web.api import utils from swh.web.api.apiurls import api_route from swh.web.api.views.utils import ( - _api_lookup, _doc_exc_id_not_found, _doc_header_link, - _doc_arg_last_elt, _doc_arg_per_page, _doc_exc_bad_id, - _doc_arg_content_id + api_lookup, doc_exc_id_not_found, doc_header_link, + doc_arg_last_elt, doc_arg_per_page, doc_exc_bad_id, + doc_arg_content_id ) @api_route(r'/content/(?P.+)/provenance/', 'content-provenance') @api_doc.route('/content/provenance/', tags=['hidden']) @api_doc.arg('q', default='sha1_git:88b9b366facda0b5ff8d8640ee9279bed346f242', argtype=api_doc.argtypes.algo_and_hash, - argdoc=_doc_arg_content_id) -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) + argdoc=doc_arg_content_id) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""List of provenance information (dict) for the matched content.""") def api_content_provenance(request, q): """Return content's provenance information if any. """ def _enrich_revision(provenance): p = provenance.copy() p['revision_url'] = \ reverse('revision', kwargs={'sha1_git': provenance['revision']}) p['content_url'] = \ reverse('content', kwargs={'q': 'sha1_git:%s' % provenance['content']}) p['origin_url'] = \ reverse('origin', kwargs={'origin_id': provenance['origin']}) p['origin_visits_url'] = \ reverse('origin-visits', kwargs={'origin_id': provenance['origin']}) p['origin_visit_url'] = \ reverse('origin-visit', kwargs={'origin_id': provenance['origin'], 'visit_id': provenance['visit']}) return p - return _api_lookup( + return api_lookup( service.lookup_content_provenance, q, notfound_msg='Content with {} not found.'.format(q), enrich_fn=_enrich_revision) @api_route(r'/content/(?P.+)/filetype/', 'content-filetype') @api_doc.route('/content/filetype/') @api_doc.arg('q', default='sha1:1fc6129a692e7a87b5450e2ba56e7669d0c5775d', argtype=api_doc.argtypes.algo_and_hash, - argdoc=_doc_arg_content_id) -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) + argdoc=doc_arg_content_id) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""Filetype information (dict) for the matched content.""") def api_content_filetype(request, q): """Get information about the detected MIME type of a content object. """ - return _api_lookup( + return api_lookup( service.lookup_content_filetype, q, notfound_msg='No filetype information found for content {}.'.format(q), enrich_fn=utils.enrich_metadata_endpoint) @api_route(r'/content/(?P.+)/language/', 'content-language') @api_doc.route('/content/language/') @api_doc.arg('q', default='sha1:1fc6129a692e7a87b5450e2ba56e7669d0c5775d', argtype=api_doc.argtypes.algo_and_hash, - argdoc=_doc_arg_content_id) -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) + argdoc=doc_arg_content_id) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""Language information (dict) for the matched content.""") def api_content_language(request, q): """Get information about the detected (programming) language of a content object. """ - return _api_lookup( + return api_lookup( service.lookup_content_language, q, notfound_msg='No language information found for content {}.'.format(q), enrich_fn=utils.enrich_metadata_endpoint) @api_route(r'/content/(?P.+)/license/', 'content-license') @api_doc.route('/content/license/') @api_doc.arg('q', default='sha1:1fc6129a692e7a87b5450e2ba56e7669d0c5775d', argtype=api_doc.argtypes.algo_and_hash, - argdoc=_doc_arg_content_id) -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) + argdoc=doc_arg_content_id) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""License information (dict) for the matched content.""") def api_content_license(request, q): """Get information about the detected license of a content object. """ - return _api_lookup( + return api_lookup( service.lookup_content_license, q, notfound_msg='No license information found for content {}.'.format(q), enrich_fn=utils.enrich_metadata_endpoint) @api_route(r'/content/(?P.+)/ctags/', 'content-ctags') @api_doc.route('/content/ctags/', tags=['upcoming']) @api_doc.arg('q', default='sha1:1fc6129a692e7a87b5450e2ba56e7669d0c5775d', argtype=api_doc.argtypes.algo_and_hash, - argdoc=_doc_arg_content_id) -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) + argdoc=doc_arg_content_id) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""Ctags symbol (dict) for the matched content.""") def api_content_ctags(request, q): """Get information about all `Ctags `_-style symbols defined in a content object. """ - return _api_lookup( + return api_lookup( service.lookup_content_ctags, q, notfound_msg='No ctags symbol found for content {}.'.format(q), enrich_fn=utils.enrich_metadata_endpoint) @api_route(r'/content/(?P.+)/raw/', 'content-raw') @api_doc.route('/content/raw/', handle_response=True) @api_doc.arg('q', default='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', argtype=api_doc.argtypes.algo_and_hash, - argdoc=_doc_arg_content_id) + argdoc=doc_arg_content_id) @api_doc.param('filename', default=None, argtype=api_doc.argtypes.str, doc='User\'s desired filename. If provided, the downloaded' ' content will get that filename.') -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.octet_stream, retdoc='The raw content data as an octet stream') def api_content_raw(request, q): """Get the raw content of a content object (AKA "blob"), as a byte sequence. """ def generate(content): yield content['data'] content_raw = service.lookup_content_raw(q) if not content_raw: raise NotFoundExc('Content %s is not found.' % q) content_filetype = service.lookup_content_filetype(q) if not content_filetype: raise NotFoundExc('Content %s is not available for download.' % q) mimetype = content_filetype['mimetype'] if 'text/' not in mimetype: raise ForbiddenExc('Only textual content is available for download. ' 'Actual content mimetype is %s.' % mimetype) filename = utils.get_query_params(request).get('filename') if not filename: filename = 'content_%s_raw' % q.replace(':', '_') response = HttpResponse(generate(content_raw), content_type='application/octet-stream') response['Content-disposition'] = 'attachment; filename=%s' % filename return response @api_route(r'/content/symbol/(?P.+)/', 'content-symbol') @api_doc.route('/content/symbol/', tags=['upcoming']) @api_doc.arg('q', default='hello', argtype=api_doc.argtypes.str, argdoc="""An expression string to lookup in swh's raw content""") -@api_doc.header('Link', doc=_doc_header_link) +@api_doc.header('Link', doc=doc_header_link) @api_doc.param('last_sha1', default=None, argtype=api_doc.argtypes.str, - doc=_doc_arg_last_elt) + doc=doc_arg_last_elt) @api_doc.param('per_page', default=10, argtype=api_doc.argtypes.int, - doc=_doc_arg_per_page) + doc=doc_arg_per_page) @api_doc.returns(rettype=api_doc.rettypes.list, retdoc="""A list of dict whose content matches the expression. Each dict has the following keys: - id (bytes): identifier of the content - name (text): symbol whose content match the expression - kind (text): kind of the symbol that matched - lang (text): Language for that entry - line (int): Number line for the symbol """) def api_content_symbol(request, q=None): """Search content objects by `Ctags `_-style symbol (e.g., function name, data type, method, ...). """ result = {} last_sha1 = utils.get_query_params(request).get('last_sha1', None) per_page = int(utils.get_query_params(request).get('per_page', '10')) def lookup_exp(exp, last_sha1=last_sha1, per_page=per_page): return service.lookup_expression(exp, last_sha1, per_page) - symbols = _api_lookup( + symbols = api_lookup( lookup_exp, q, notfound_msg="No indexed raw content match expression '{}'.".format(q), enrich_fn=functools.partial(utils.enrich_content, top_url=True)) if symbols: l = len(symbols) if l == per_page: query_params = {} new_last_sha1 = symbols[-1]['sha1'] query_params['last_sha1'] = new_last_sha1 if utils.get_query_params(request).get('per_page'): query_params['per_page'] = per_page result['headers'] = { 'link-next': reverse('content-symbol', kwargs={'q': q}, query_params=query_params) } result.update({ 'results': symbols }) return result @api_route(r'/content/known/search/', 'content-known', methods=['POST']) @api_route(r'/content/known/(?P(?!search).*)/', 'content-known') @api_doc.route('/content/known/', tags=['hidden']) @api_doc.arg('q', default='adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', argtype=api_doc.argtypes.sha1, argdoc='content identifier as a sha1 checksum') @api_doc.param('q', default=None, argtype=api_doc.argtypes.str, doc="""(POST request) An algo_hash:hash string, where algo_hash is one of sha1, sha1_git or sha256 and hash is the hash to search for in SWH""") -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""a dictionary with results (found/not found for each given identifier) and statistics about how many identifiers were found""") def api_check_content_known(request, q=None): """Check whether some content (AKA "blob") is present in the archive. Lookup can be performed by various means: - a GET request with one or several hashes, separated by ',' - a POST request with one or several hashes, passed as (multiple) values for parameter 'q' """ response = {'search_res': None, 'search_stats': None} search_stats = {'nbfiles': 0, 'pct': 0} search_res = None queries = [] # GET: Many hash separated values request if q: hashes = q.split(',') for v in hashes: queries.append({'filename': None, 'sha1': v}) # POST: Many hash requests in post form submission elif request.method == 'POST': data = request.data if hasattr(request, 'data') else request.DATA # Remove potential inputs with no associated value for k, v in data.items(): if v is not None: if k == 'q' and len(v) > 0: queries.append({'filename': None, 'sha1': v}) elif v != '': queries.append({'filename': k, 'sha1': v}) if queries: lookup = service.lookup_multiple_hashes(queries) result = [] l = len(queries) for el in lookup: res_d = {'sha1': el['sha1'], 'found': el['found']} if 'filename' in el and el['filename']: res_d['filename'] = el['filename'] result.append(res_d) search_res = result nbfound = len([x for x in lookup if x['found']]) search_stats['nbfiles'] = l search_stats['pct'] = (nbfound / l) * 100 response['search_res'] = search_res response['search_stats'] = search_stats return response @api_route(r'/content/(?P.+)/', 'content') @api_doc.route('/content/') @api_doc.arg('q', default='dc2830a9e72f23c1dfebef4413003221baa5fb62', argtype=api_doc.argtypes.algo_and_hash, - argdoc=_doc_arg_content_id) -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) + argdoc=doc_arg_content_id) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""known metadata for content identified by q""") def api_content_metadata(request, q): """Get information about a content (AKA "blob") object. """ - return _api_lookup( + return api_lookup( service.lookup_content, q, notfound_msg='Content with {} not found.'.format(q), enrich_fn=functools.partial(utils.enrich_content, query_string=q)) diff --git a/swh/web/api/views/directory.py b/swh/web/api/views/directory.py index d7e9324a..0594b7d2 100644 --- a/swh/web/api/views/directory.py +++ b/swh/web/api/views/directory.py @@ -1,59 +1,59 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api import utils from swh.web.api import apidoc as api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import ( - _api_lookup, _doc_exc_id_not_found, - _doc_exc_bad_id, + api_lookup, doc_exc_id_not_found, + doc_exc_bad_id, ) @api_route(r'/directory/(?P[0-9a-f]+)/', 'directory') @api_route(r'/directory/(?P[0-9a-f]+)/(?P.+)/', 'directory') @api_doc.route('/directory/') @api_doc.arg('sha1_git', default='1bd0e65f7d2ff14ae994de17a1e7fe65111dcad8', argtype=api_doc.argtypes.sha1_git, argdoc='directory identifier') @api_doc.arg('path', default='codec/demux', argtype=api_doc.argtypes.path, argdoc='path relative to directory identified by sha1_git') -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""either a list of directory entries with their metadata, or the metadata of a single directory entry""") def api_directory(request, sha1_git, path=None): """Get information about directory or directory entry objects. Directories are identified by SHA1 checksums, compatible with Git directory identifiers. See ``directory_identifier`` in our `data model module `_ for details about how they are computed. When given only a directory identifier, this endpoint returns information about the directory itself, returning its content (usually a list of directory entries). When given a directory identifier and a path, this endpoint returns information about the directory entry pointed by the relative path, starting path resolution from the given directory. """ if path: error_msg_path = ('Entry with path %s relative to directory ' 'with sha1_git %s not found.') % (path, sha1_git) - return _api_lookup( + return api_lookup( service.lookup_directory_with_path, sha1_git, path, notfound_msg=error_msg_path, enrich_fn=utils.enrich_directory) else: error_msg_nopath = 'Directory with sha1_git %s not found.' % sha1_git - return _api_lookup( + return api_lookup( service.lookup_directory, sha1_git, notfound_msg=error_msg_nopath, enrich_fn=utils.enrich_directory) diff --git a/swh/web/api/views/entity.py b/swh/web/api/views/entity.py index ae4022c4..06ca2316 100644 --- a/swh/web/api/views/entity.py +++ b/swh/web/api/views/entity.py @@ -1,33 +1,33 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api import utils from swh.web.api import apidoc as api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import ( - _api_lookup, _doc_exc_id_not_found, - _doc_exc_bad_id + api_lookup, doc_exc_id_not_found, + doc_exc_bad_id ) @api_route(r'/entity/(?P.+)/', 'entity') @api_doc.route('/entity/', tags=['hidden']) @api_doc.arg('uuid', default='5f4d4c51-498a-4e28-88b3-b3e4e8396cba', argtype=api_doc.argtypes.uuid, argdoc="The entity's uuid identifier") -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc='The metadata of the entity identified by uuid') def api_entity_by_uuid(request, uuid): """Return content information if content is found. """ - return _api_lookup( + return api_lookup( service.lookup_entity_by_uuid, uuid, notfound_msg="Entity with uuid '%s' not found." % uuid, enrich_fn=utils.enrich_entity) diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py index 8c126a7d..0299abe1 100644 --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -1,179 +1,179 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.common.utils import reverse from swh.web.api import utils from swh.web.api import apidoc as api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import ( - _api_lookup, _doc_exc_id_not_found, _doc_header_link, - _doc_arg_last_elt, _doc_arg_per_page + api_lookup, doc_exc_id_not_found, doc_header_link, + doc_arg_last_elt, doc_arg_per_page ) @api_route(r'/origin/(?P[0-9]+)/', 'origin') @api_route(r'/origin/(?P[a-z]+)/url/(?P.+)', 'origin') @api_doc.route('/origin/') @api_doc.arg('origin_id', default=1, argtype=api_doc.argtypes.int, argdoc='origin identifier (when looking up by ID)') @api_doc.arg('origin_type', default='git', argtype=api_doc.argtypes.str, argdoc='origin type (when looking up by type+URL)') @api_doc.arg('origin_url', default='https://github.com/hylang/hy', argtype=api_doc.argtypes.path, argdoc='origin URL (when looking up by type+URL)') -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""The metadata of the origin corresponding to the given criteria""") def api_origin(request, origin_id=None, origin_type=None, origin_url=None): """Get information about a software origin. Software origins might be looked up by origin type and canonical URL (e.g., "git" + a "git clone" URL), or by their unique (but otherwise meaningless) identifier. """ ori_dict = { 'id': origin_id, 'type': origin_type, 'url': origin_url } ori_dict = {k: v for k, v in ori_dict.items() if ori_dict[k]} if 'id' in ori_dict: error_msg = 'Origin with id %s not found.' % ori_dict['id'] else: error_msg = 'Origin with type %s and URL %s not found' % ( ori_dict['type'], ori_dict['url']) def _enrich_origin(origin): if 'id' in origin: o = origin.copy() o['origin_visits_url'] = \ reverse('origin-visits', kwargs={'origin_id': origin['id']}) return o return origin - return _api_lookup( + return api_lookup( service.lookup_origin, ori_dict, notfound_msg=error_msg, enrich_fn=_enrich_origin) @api_route(r'/origin/(?P[0-9]+)/visits/', 'origin-visits') @api_doc.route('/origin/visits/') @api_doc.arg('origin_id', default=1, argtype=api_doc.argtypes.int, argdoc='software origin identifier') -@api_doc.header('Link', doc=_doc_header_link) +@api_doc.header('Link', doc=doc_header_link) @api_doc.param('last_visit', default=None, argtype=api_doc.argtypes.int, - doc=_doc_arg_last_elt) + doc=doc_arg_last_elt) @api_doc.param('per_page', default=10, argtype=api_doc.argtypes.int, - doc=_doc_arg_per_page) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) + doc=doc_arg_per_page) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.list, retdoc="""a list of dictionaries describing individual visits. For each visit, its identifier, timestamp (as UNIX time), outcome, and visit-specific URL for more information are given.""") def api_origin_visits(request, origin_id): """Get information about all visits of a given software origin. """ result = {} per_page = int(utils.get_query_params(request).get('per_page', '10')) last_visit = utils.get_query_params(request).get('last_visit') if last_visit: last_visit = int(last_visit) def _lookup_origin_visits( origin_id, last_visit=last_visit, per_page=per_page): return service.lookup_origin_visits( origin_id, last_visit=last_visit, per_page=per_page) def _enrich_origin_visit(origin_visit): ov = origin_visit.copy() ov['origin_visit_url'] = reverse('origin-visit', kwargs={'origin_id': origin_id, 'visit_id': ov['visit']}) return ov - r = _api_lookup( + r = api_lookup( _lookup_origin_visits, origin_id, notfound_msg='No origin {} found'.format(origin_id), enrich_fn=_enrich_origin_visit) if r: l = len(r) if l == per_page: new_last_visit = r[-1]['visit'] query_params = {} query_params['last_visit'] = new_last_visit if utils.get_query_params(request).get('per_page'): query_params['per_page'] = per_page result['headers'] = { 'link-next': reverse('origin-visits', kwargs={'origin_id': origin_id}, query_params=query_params) } result.update({ 'results': r }) return result @api_route(r'/origin/(?P[0-9]+)/visit/(?P[0-9]+)/', 'origin-visit') @api_doc.route('/origin/visit/') @api_doc.arg('origin_id', default=1, argtype=api_doc.argtypes.int, argdoc='software origin identifier') @api_doc.arg('visit_id', default=1, argtype=api_doc.argtypes.int, argdoc="""visit identifier, relative to the origin identified by origin_id""") -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""dictionary containing both metadata for the entire visit (e.g., timestamp as UNIX time, visit outcome, etc.) and what was at the software origin during the visit (i.e., a mapping from branches to other archive objects)""") def api_origin_visit(request, origin_id, visit_id): """Get information about a specific visit of a software origin. """ def _enrich_origin_visit(origin_visit): ov = origin_visit.copy() ov['origin_url'] = reverse('origin', kwargs={'origin_id': ov['origin']}) if 'occurrences' in ov: ov['occurrences'] = { k: utils.enrich_object(v) for k, v in ov['occurrences'].items() } return ov - return _api_lookup( + return api_lookup( service.lookup_origin_visit, origin_id, visit_id, notfound_msg=('No visit {} for origin {} found' .format(visit_id, origin_id)), enrich_fn=_enrich_origin_visit) diff --git a/swh/web/api/views/person.py b/swh/web/api/views/person.py index feb2d06a..98c5f454 100644 --- a/swh/web/api/views/person.py +++ b/swh/web/api/views/person.py @@ -1,29 +1,29 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api import apidoc as api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import ( - _api_lookup, _doc_exc_id_not_found, + api_lookup, doc_exc_id_not_found, ) @api_route(r'/person/(?P[0-9]+)/', 'person') @api_doc.route('/person/') @api_doc.arg('person_id', default=42, argtype=api_doc.argtypes.int, argdoc='person identifier') -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc='The metadata of the person identified by person_id') def api_person(request, person_id): """Get information about a person. """ - return _api_lookup( + return api_lookup( service.lookup_person, person_id, notfound_msg='Person with id {} not found.'.format(person_id)) diff --git a/swh/web/api/views/release.py b/swh/web/api/views/release.py index 58f4456d..8cbf4e91 100644 --- a/swh/web/api/views/release.py +++ b/swh/web/api/views/release.py @@ -1,38 +1,38 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api import utils from swh.web.api import apidoc as api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import ( - _api_lookup, _doc_exc_id_not_found, _doc_exc_bad_id + api_lookup, doc_exc_id_not_found, doc_exc_bad_id ) @api_route(r'/release/(?P[0-9a-f]+)/', 'release') @api_doc.route('/release/') @api_doc.arg('sha1_git', default='7045404f3d1c54e6473c71bbb716529fbad4be24', argtype=api_doc.argtypes.sha1_git, argdoc='release identifier') -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc='The metadata of the release identified by sha1_git') def api_release(request, sha1_git): """Get information about a release. Releases are identified by SHA1 checksums, compatible with Git tag identifiers. See ``release_identifier`` in our `data model module `_ for details about how they are computed. """ error_msg = 'Release with sha1_git %s not found.' % sha1_git - return _api_lookup( + return api_lookup( service.lookup_release, sha1_git, notfound_msg=error_msg, enrich_fn=utils.enrich_release) diff --git a/swh/web/api/views/revision.py b/swh/web/api/views/revision.py index faf939cd..e8c02088 100644 --- a/swh/web/api/views/revision.py +++ b/swh/web/api/views/revision.py @@ -1,421 +1,421 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.http import HttpResponse from swh.web.common import service from swh.web.common.utils import reverse from swh.web.common.utils import parse_timestamp from swh.web.api import utils from swh.web.api import apidoc as api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import ( - _api_lookup, _doc_exc_id_not_found, _doc_header_link, - _doc_arg_per_page, _doc_exc_bad_id, - _doc_ret_revision_log, _doc_ret_revision_meta + api_lookup, doc_exc_id_not_found, doc_header_link, + doc_arg_per_page, doc_exc_bad_id, + doc_ret_revision_log, doc_ret_revision_meta ) def _revision_directory_by(revision, path, request_path, limit=100, with_data=False): """Compute the revision matching criterion's directory or content data. Args: revision: dictionary of criterions representing a revision to lookup path: directory's path to lookup request_path: request path which holds the original context to limit: optional query parameter to limit the revisions log (default to 100). For now, note that this limit could impede the transitivity conclusion about sha1_git not being an ancestor of with_data: indicate to retrieve the content's raw data if path resolves to a content. """ def enrich_directory_local(dir, context_url=request_path): return utils.enrich_directory(dir, context_url) rev_id, result = service.lookup_directory_through_revision( revision, path, limit=limit, with_data=with_data) content = result['content'] if result['type'] == 'dir': # dir_entries result['content'] = list(map(enrich_directory_local, content)) else: # content result['content'] = utils.enrich_content(content) return result @api_route(r'/revision/origin/(?P[0-9]+)' r'/branch/(?P.+)/log/', 'revision-origin-log') @api_route(r'/revision/origin/(?P[0-9]+)/log/', 'revision-origin-log') @api_route(r'/revision/origin/(?P[0-9]+)' r'/ts/(?P.+)/log/', 'revision-origin-log') @api_route(r'/revision/origin/(?P[0-9]+)' r'/branch/(?P.+)' r'/ts/(?P.+)/log/', 'revision-origin-log') @api_doc.route('/revision/origin/log/') @api_doc.arg('origin_id', default=1, argtype=api_doc.argtypes.int, argdoc="The revision's SWH origin identifier") @api_doc.arg('branch_name', default='refs/heads/master', argtype=api_doc.argtypes.path, argdoc="""(Optional) The revision's branch name within the origin specified. Defaults to 'refs/heads/master'.""") @api_doc.arg('ts', default='2000-01-17T11:23:54+00:00', argtype=api_doc.argtypes.ts, argdoc="""(Optional) A time or timestamp string to parse""") -@api_doc.header('Link', doc=_doc_header_link) +@api_doc.header('Link', doc=doc_header_link) @api_doc.param('per_page', default=10, argtype=api_doc.argtypes.int, - doc=_doc_arg_per_page) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) -@api_doc.returns(rettype=api_doc.rettypes.dict, retdoc=_doc_ret_revision_log) + doc=doc_arg_per_page) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) +@api_doc.returns(rettype=api_doc.rettypes.dict, retdoc=doc_ret_revision_log) def api_revision_log_by(request, origin_id, branch_name='refs/heads/master', ts=None): """Show the commit log for a revision, searching for it based on software origin, branch name, and/or visit timestamp. This endpoint behaves like ``/log``, but operates on the revision that has been found at a given software origin, close to a given point in time, pointed by a given branch. """ result = {} per_page = int(utils.get_query_params(request).get('per_page', '10')) if ts: ts = parse_timestamp(ts) def lookup_revision_log_by_with_limit(o_id, br, ts, limit=per_page+1): return service.lookup_revision_log_by(o_id, br, ts, limit) error_msg = 'No revision matching origin %s ' % origin_id error_msg += ', branch name %s' % branch_name error_msg += (' and time stamp %s.' % ts) if ts else '.' - rev_get = _api_lookup( + rev_get = api_lookup( lookup_revision_log_by_with_limit, origin_id, branch_name, ts, notfound_msg=error_msg, enrich_fn=utils.enrich_revision) l = len(rev_get) if l == per_page+1: revisions = rev_get[:-1] last_sha1_git = rev_get[-1]['id'] params = {k: v for k, v in {'origin_id': origin_id, 'branch_name': branch_name, 'ts': ts, }.items() if v is not None} query_params = {} query_params['sha1_git'] = last_sha1_git if utils.get_query_params(request).get('per_page'): query_params['per_page'] = per_page result['headers'] = { 'link-next': reverse('revision-origin-log', kwargs=params, query_params=query_params) } else: revisions = rev_get result.update({'results': revisions}) return result @api_route(r'/revision/origin/(?P[0-9]+)/directory/', 'revision-directory') @api_route(r'/revision/origin/(?P[0-9]+)/directory/(?P.+)/', 'revision-directory') @api_route(r'/revision/origin/(?P[0-9]+)' r'/branch/(?P.+)/directory/', 'revision-directory') @api_route(r'/revision/origin/(?P[0-9]+)' r'/branch/(?P.+)/ts/(?P.+)/directory/', 'revision-directory') @api_route(r'/revision/origin/(?P[0-9]+)' r'/branch/(?P.+)/directory/(?P.+)/', 'revision-directory') @api_route(r'/revision/origin/(?P[0-9]+)' r'/branch/(?P.+)/ts/(?P.+)' r'/directory/(?P.+)/', 'revision-directory') @api_doc.route('/revision/origin/directory/', tags=['hidden']) @api_doc.arg('origin_id', default=1, argtype=api_doc.argtypes.int, argdoc="The revision's origin's SWH identifier") @api_doc.arg('branch_name', default='refs/heads/master', argtype=api_doc.argtypes.path, argdoc="""The optional branch for the given origin (default to master""") @api_doc.arg('ts', default='2000-01-17T11:23:54+00:00', argtype=api_doc.argtypes.ts, argdoc="""Optional timestamp (default to the nearest time crawl of timestamp)""") @api_doc.arg('path', default='Dockerfile', argtype=api_doc.argtypes.path, argdoc='The path to the directory or file to display') -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""The metadata of the revision corresponding to the given criteria""") def api_directory_through_revision_origin(request, origin_id, branch_name="refs/heads/master", ts=None, path=None, with_data=False): """Display directory or content information through a revision identified by origin/branch/timestamp. """ if ts: ts = parse_timestamp(ts) return _revision_directory_by({'origin_id': origin_id, 'branch_name': branch_name, 'ts': ts }, path, request.path, with_data=with_data) @api_route(r'/revision/origin/(?P[0-9]+)/', 'revision-origin') @api_route(r'/revision/origin/(?P[0-9]+)' r'/branch/(?P.+)/', 'revision-origin') @api_route(r'/revision/origin/(?P[0-9]+)' r'/branch/(?P.+)/ts/(?P.+)/', 'revision-origin') @api_route(r'/revision/origin/(?P[0-9]+)/ts/(?P.+)/', 'revision-origin') @api_doc.route('/revision/origin/') @api_doc.arg('origin_id', default=1, argtype=api_doc.argtypes.int, argdoc='software origin identifier') @api_doc.arg('branch_name', default='refs/heads/master', argtype=api_doc.argtypes.path, argdoc="""(optional) fully-qualified branch name, e.g., "refs/heads/master". Defaults to the master branch.""") @api_doc.arg('ts', default=None, argtype=api_doc.argtypes.ts, argdoc="""(optional) timestamp close to which the revision pointed by the given branch should be looked up. Defaults to now.""") -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) -@api_doc.returns(rettype=api_doc.rettypes.dict, retdoc=_doc_ret_revision_meta) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) +@api_doc.returns(rettype=api_doc.rettypes.dict, retdoc=doc_ret_revision_meta) def api_revision_with_origin(request, origin_id, branch_name="refs/heads/master", ts=None): """Get information about a revision, searching for it based on software origin, branch name, and/or visit timestamp. This endpoint behaves like ``/revision``, but operates on the revision that has been found at a given software origin, close to a given point in time, pointed by a given branch. """ ts = parse_timestamp(ts) - return _api_lookup( + return api_lookup( service.lookup_revision_by, origin_id, branch_name, ts, notfound_msg=('Revision with (origin_id: {}, branch_name: {}' ', ts: {}) not found.'.format(origin_id, branch_name, ts)), enrich_fn=utils.enrich_revision) @api_route(r'/revision/(?P[0-9a-f]+)/prev/(?P[0-9a-f/]+)/', 'revision-context') @api_doc.route('/revision/prev/', tags=['hidden']) @api_doc.arg('sha1_git', default='ec72c666fb345ea5f21359b7bc063710ce558e39', argtype=api_doc.argtypes.sha1_git, argdoc="The revision's sha1_git identifier") @api_doc.arg('context', default='6adc4a22f20bbf3bbc754f1ec8c82be5dfb5c71a', argtype=api_doc.argtypes.path, argdoc='The navigation breadcrumbs -- use at your own risk') -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc='The metadata of the revision identified by sha1_git') def api_revision_with_context(request, sha1_git, context): """Return information about revision with id sha1_git. """ def _enrich_revision(revision, context=context): return utils.enrich_revision(revision, context) - return _api_lookup( + return api_lookup( service.lookup_revision, sha1_git, notfound_msg='Revision with sha1_git %s not found.' % sha1_git, enrich_fn=_enrich_revision) @api_route(r'/revision/(?P[0-9a-f]+)/', 'revision') @api_doc.route('/revision/') @api_doc.arg('sha1_git', default='aafb16d69fd30ff58afdd69036a26047f3aebdc6', argtype=api_doc.argtypes.sha1_git, argdoc="revision identifier") -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) -@api_doc.returns(rettype=api_doc.rettypes.dict, retdoc=_doc_ret_revision_meta) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) +@api_doc.returns(rettype=api_doc.rettypes.dict, retdoc=doc_ret_revision_meta) def api_revision(request, sha1_git): """Get information about a revision. Revisions are identified by SHA1 checksums, compatible with Git commit identifiers. See ``revision_identifier`` in our `data model module `_ for details about how they are computed. """ - return _api_lookup( + return api_lookup( service.lookup_revision, sha1_git, notfound_msg='Revision with sha1_git {} not found.'.format(sha1_git), enrich_fn=utils.enrich_revision) @api_route(r'/revision/(?P[0-9a-f]+)/raw/', 'revision-raw-message') @api_doc.route('/revision/raw/', tags=['hidden'], handle_response=True) @api_doc.arg('sha1_git', default='ec72c666fb345ea5f21359b7bc063710ce558e39', argtype=api_doc.argtypes.sha1_git, argdoc="The queried revision's sha1_git identifier") -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.octet_stream, retdoc="""The message of the revision identified by sha1_git as a downloadable octet stream""") def api_revision_raw_message(request, sha1_git): """Return the raw data of the message of revision identified by sha1_git """ raw = service.lookup_revision_message(sha1_git) response = HttpResponse(raw['message'], content_type='application/octet-stream') response['Content-disposition'] = \ 'attachment;filename=rev_%s_raw' % sha1_git return response @api_route(r'/revision/(?P[0-9a-f]+)/directory/', 'revision-directory') @api_route(r'/revision/(?P[0-9a-f]+)/directory/(?P.+)/', 'revision-directory') @api_doc.route('/revision/directory/') @api_doc.arg('sha1_git', default='ec72c666fb345ea5f21359b7bc063710ce558e39', argtype=api_doc.argtypes.sha1_git, argdoc='revision identifier') @api_doc.arg('dir_path', default='Documentation/BUG-HUNTING', argtype=api_doc.argtypes.path, argdoc="""path relative to the root directory of revision identifier by sha1_git""") -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) @api_doc.returns(rettype=api_doc.rettypes.dict, retdoc="""either a list of directory entries with their metadata, or the metadata of a single directory entry""") def api_revision_directory(request, sha1_git, dir_path=None, with_data=False): """Get information about directory (entry) objects associated to revisions. Each revision is associated to a single "root" directory. This endpoint behaves like ``/directory/``, but operates on the root directory associated to a given revision. """ return _revision_directory_by({'sha1_git': sha1_git}, dir_path, request.path, with_data=with_data) @api_route(r'/revision/(?P[0-9a-f]+)/log/', 'revision-log') @api_route(r'/revision/(?P[0-9a-f]+)' r'/prev/(?P[0-9a-f/]+)/log/', 'revision-log') @api_doc.route('/revision/log/') @api_doc.arg('sha1_git', default='37fc9e08d0c4b71807a4f1ecb06112e78d91c283', argtype=api_doc.argtypes.sha1_git, argdoc='revision identifier') @api_doc.arg('prev_sha1s', default='6adc4a22f20bbf3bbc754f1ec8c82be5dfb5c71a', argtype=api_doc.argtypes.path, argdoc="""(Optional) Navigation breadcrumbs (descendant revisions previously visited). If multiple values, use / as delimiter. """) -@api_doc.header('Link', doc=_doc_header_link) +@api_doc.header('Link', doc=doc_header_link) @api_doc.param('per_page', default=10, argtype=api_doc.argtypes.int, - doc=_doc_arg_per_page) -@api_doc.raises(exc=api_doc.excs.badinput, doc=_doc_exc_bad_id) -@api_doc.raises(exc=api_doc.excs.notfound, doc=_doc_exc_id_not_found) -@api_doc.returns(rettype=api_doc.rettypes.dict, retdoc=_doc_ret_revision_log) + doc=doc_arg_per_page) +@api_doc.raises(exc=api_doc.excs.badinput, doc=doc_exc_bad_id) +@api_doc.raises(exc=api_doc.excs.notfound, doc=doc_exc_id_not_found) +@api_doc.returns(rettype=api_doc.rettypes.dict, retdoc=doc_ret_revision_log) def api_revision_log(request, sha1_git, prev_sha1s=None): """Get a list of all revisions heading to a given one, i.e., show the commit log. """ result = {} per_page = int(utils.get_query_params(request).get('per_page', '10')) def lookup_revision_log_with_limit(s, limit=per_page+1): return service.lookup_revision_log(s, limit) error_msg = 'Revision with sha1_git %s not found.' % sha1_git - rev_get = _api_lookup(lookup_revision_log_with_limit, sha1_git, - notfound_msg=error_msg, - enrich_fn=utils.enrich_revision) + rev_get = api_lookup(lookup_revision_log_with_limit, sha1_git, + notfound_msg=error_msg, + enrich_fn=utils.enrich_revision) l = len(rev_get) if l == per_page+1: rev_backward = rev_get[:-1] new_last_sha1 = rev_get[-1]['id'] query_params = {} if utils.get_query_params(request).get('per_page'): query_params['per_page'] = per_page result['headers'] = { 'link-next': reverse('revision-log', kwargs={'sha1_git': new_last_sha1}, query_params=query_params) } else: rev_backward = rev_get if not prev_sha1s: # no nav breadcrumbs, so we're done revisions = rev_backward else: rev_forward_ids = prev_sha1s.split('/') - rev_forward = _api_lookup( + rev_forward = api_lookup( service.lookup_revision_multiple, rev_forward_ids, notfound_msg=error_msg, enrich_fn=utils.enrich_revision) revisions = rev_forward + rev_backward result.update({ 'results': revisions }) return result diff --git a/swh/web/api/views/utils.py b/swh/web/api/views/utils.py index 7a267b9f..4d44f210 100644 --- a/swh/web/api/views/utils.py +++ b/swh/web/api/views/utils.py @@ -1,90 +1,90 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from rest_framework.response import Response from rest_framework.decorators import api_view from types import GeneratorType from swh.web.common.exc import NotFoundExc from swh.web.api.apiurls import APIUrls, api_route # canned doc string snippets that are used in several doc strings -_doc_arg_content_id = """A "[hash_type:]hash" content identifier, where +doc_arg_content_id = """A "[hash_type:]hash" content identifier, where hash_type is one of "sha1" (the default), "sha1_git", "sha256", and hash is a checksum obtained with the hash_type hashing algorithm.""" -_doc_arg_last_elt = 'element to start listing from, for pagination purposes' -_doc_arg_per_page = 'number of elements to list, for pagination purposes' +doc_arg_last_elt = 'element to start listing from, for pagination purposes' +doc_arg_per_page = 'number of elements to list, for pagination purposes' -_doc_exc_bad_id = 'syntax error in the given identifier(s)' -_doc_exc_id_not_found = 'no object matching the given criteria could be found' +doc_exc_bad_id = 'syntax error in the given identifier(s)' +doc_exc_id_not_found = 'no object matching the given criteria could be found' -_doc_ret_revision_meta = 'metadata of the revision identified by sha1_git' -_doc_ret_revision_log = """list of dictionaries representing the metadata of +doc_ret_revision_meta = 'metadata of the revision identified by sha1_git' +doc_ret_revision_log = """list of dictionaries representing the metadata of each revision found in the commit log heading to revision sha1_git. For each commit at least the following information are returned: author/committer, authoring/commit timestamps, revision id, commit message, parent (i.e., immediately preceding) commits, "root" directory id.""" -_doc_header_link = """indicates that a subsequent result page is available, +doc_header_link = """indicates that a subsequent result page is available, pointing to it""" -def _api_lookup(lookup_fn, *args, - notfound_msg='Object not found', - enrich_fn=lambda x: x): +def api_lookup(lookup_fn, *args, + notfound_msg='Object not found', + enrich_fn=lambda x: x): """Capture a redundant behavior of: - looking up the backend with a criteria (be it an identifier or checksum) passed to the function lookup_fn - if nothing is found, raise an NotFoundExc exception with error message notfound_msg. - Otherwise if something is returned: - either as list, map or generator, map the enrich_fn function to it and return the resulting data structure as list. - either as dict and pass to enrich_fn and return the dict enriched. Args: - criteria: discriminating criteria to lookup - lookup_fn: function expects one criteria and optional supplementary *args. - notfound_msg: if nothing matching the criteria is found, raise NotFoundExc with this error message. - enrich_fn: Function to use to enrich the result returned by lookup_fn. Default to the identity function if not provided. - *args: supplementary arguments to pass to lookup_fn. Raises: NotFoundExp or whatever `lookup_fn` raises. """ res = lookup_fn(*args) if not res: raise NotFoundExc(notfound_msg) if isinstance(res, (map, list, GeneratorType)): return [enrich_fn(x) for x in res] return enrich_fn(res) @api_view(['GET', 'HEAD']) def api_home(request): return Response({}, template_name='api.html') APIUrls.add_url_pattern(r'^$', api_home, view_name='api_homepage') @api_route(r'/', 'endpoints') def api_endpoints(request): """Display the list of opened api endpoints. """ routes = APIUrls.get_app_endpoints().copy() for route, doc in routes.items(): doc['doc_intro'] = doc['docstring'].split('\n\n')[0] # Return a list of routes with consistent ordering env = { 'doc_routes': sorted(routes.items()) } return Response(env, template_name="api-endpoints.html") diff --git a/swh/web/doc_config.py b/swh/web/doc_config.py index 87cf4a4c..948c0a80 100644 --- a/swh/web/doc_config.py +++ b/swh/web/doc_config.py @@ -1,25 +1,34 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from sphinxcontrib import httpdomain _swh_web_base_url = 'https://archive.softwareheritage.org' _swh_web_api_endpoint = 'api' _swh_web_api_version = 1 _swh_web_api_url = '%s/%s/%s/' % (_swh_web_base_url, _swh_web_api_endpoint, _swh_web_api_version) _swh_web_browse_endpoint = 'browse' _swh_web_browse_url = '%s/%s/' % (_swh_web_base_url, _swh_web_browse_endpoint) def customize_sphinx_conf(sphinx_conf): + """ + Utility function used to customize the sphinx doc build for swh-web + globally (when building doc from swh-docs) or locally (when building doc + from swh-web). + + Args: + sphinx_conf (module): a reference to the sphinx conf.py module + used to build the doc. + """ # fix for sphinxcontrib.httpdomain 1.3 if 'Link' not in httpdomain.HEADER_REFS: httpdomain.HEADER_REFS['Link'] = httpdomain.IETFRef(5988, '5') sphinx_conf.extlinks['swh_web_api'] = (_swh_web_api_url + '%s', None) sphinx_conf.extlinks['swh_web_browse'] = (_swh_web_browse_url + '%s', None) diff --git a/swh/web/tests/api/test_api_lookup.py b/swh/web/tests/api/test_api_lookup.py index b8f8f5ca..cfd43e64 100644 --- a/swh/web/tests/api/test_api_lookup.py +++ b/swh/web/tests/api/test_api_lookup.py @@ -1,126 +1,126 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from nose.tools import istest from .swh_api_testcase import SWHApiTestCase from swh.web.common.exc import NotFoundExc from swh.web.api.views import utils class ApiLookupTestCase(SWHApiTestCase): @istest - def generic_api_lookup_nothing_is_found(self): + def genericapi_lookup_nothing_is_found(self): # given def test_generic_lookup_fn(sha1, another_unused_arg): assert another_unused_arg == 'unused_arg' assert sha1 == 'sha1' return None # when with self.assertRaises(NotFoundExc) as cm: - utils._api_lookup( + utils.api_lookup( test_generic_lookup_fn, 'sha1', 'unused_arg', notfound_msg='This will be raised because None is returned.') self.assertIn('This will be raised because None is returned.', cm.exception.args[0]) @istest def generic_api_map_are_enriched_and_transformed_to_list(self): # given def test_generic_lookup_fn_1(criteria0, param0, param1): assert criteria0 == 'something' return map(lambda x: x + 1, [1, 2, 3]) # when - actual_result = utils._api_lookup( + actual_result = utils.api_lookup( test_generic_lookup_fn_1, 'something', 'some param 0', 'some param 1', notfound_msg=('This is not the error message you are looking for. ' 'Move along.'), enrich_fn=lambda x: x * 2) self.assertEqual(actual_result, [4, 6, 8]) @istest def generic_api_list_are_enriched_too(self): # given def test_generic_lookup_fn_2(crit): assert crit == 'something' return ['a', 'b', 'c'] # when - actual_result = utils._api_lookup( + actual_result = utils.api_lookup( test_generic_lookup_fn_2, 'something', notfound_msg=('Not the error message you are looking for, it is. ' 'Along, you move!'), enrich_fn=lambda x: ''. join(['=', x, '='])) self.assertEqual(actual_result, ['=a=', '=b=', '=c=']) @istest def generic_api_generator_are_enriched_and_returned_as_list(self): # given def test_generic_lookup_fn_3(crit): assert crit == 'crit' return (i for i in [4, 5, 6]) # when - actual_result = utils._api_lookup( + actual_result = utils.api_lookup( test_generic_lookup_fn_3, 'crit', notfound_msg='Move!', enrich_fn=lambda x: x - 1) self.assertEqual(actual_result, [3, 4, 5]) @istest def generic_api_simple_data_are_enriched_and_returned_too(self): # given def test_generic_lookup_fn_4(crit): assert crit == '123' return {'a': 10} def test_enrich_data(x): x['a'] = x['a'] * 10 return x # when - actual_result = utils._api_lookup( + actual_result = utils.api_lookup( test_generic_lookup_fn_4, '123', notfound_msg='Nothing to do', enrich_fn=test_enrich_data) self.assertEqual(actual_result, {'a': 100}) @istest def api_lookup_not_found(self): # when with self.assertRaises(NotFoundExc) as e: - utils._api_lookup( + utils.api_lookup( lambda x: None, 'something', notfound_msg='this is the error message raised as it is None') self.assertEqual(e.exception.args[0], 'this is the error message raised as it is None') @istest def api_lookup_with_result(self): # when - actual_result = utils._api_lookup( + actual_result = utils.api_lookup( lambda x: x + '!', 'something', notfound_msg='this is the error which won\'t be used here') self.assertEqual(actual_result, 'something!') @istest def api_lookup_with_result_as_map(self): # when - actual_result = utils._api_lookup( + actual_result = utils.api_lookup( lambda x: map(lambda y: y+1, x), [1, 2, 3], notfound_msg='this is the error which won\'t be used here') self.assertEqual(actual_result, [2, 3, 4])