diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ .eggs/ __pycache__ *.egg-info/ -version.txt \ No newline at end of file +version.txt +.tox +docs/apidoc diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -15,46 +15,57 @@ Therefore to build the full documentation you need a working and complete [Software Heritage development environment][2]. -[1]: http://www.sphinx-doc.org/ -[2]: https://forge.softwareheritage.org/source/swh-environment/ How to build the doc -------------------- +Install the [Software Heritage development environment][2] + + $ git clone https://forge.softwareheritage.org/source/swh-environment + $ cd swh-environment + $ ./bin/update # this will clone needed git repos, inc. swh-doc + $ cd swh-doc + Ensure you have the required tools to generate images ([graphviz][3]'s `dot` and [plantuml][4]). On a Debian system: $ sudo apt install plantuml graphviz -[3]: https://graphviz.org -[4]: http://plantuml.com +It is also recomanded to build the doc using [tox][5], so ensure you have it +installed, eg. on a Debian system: -Then + $ sudo apt install tox - $ cd docs - $ make html -Behind the scene, this will do two things: +Then (from the `swh-environment/swh-doc/` directory): + + $ tox -e sphinx-dev + +This tox environment will build the documentation from the sources available in +the parent directory (`swh-environment`). + +Behind the scene, this tox environment will run the sphinx documentation +building process via [pifpaf][6] to encapsulate the need os Postgresql to +generate database schemas. The documentation building process itself consists +mainly in 3 steps: -### 1. Generate sphinx-apidoc rst documents for all modules +### 1. Generate documentation assets for all modules $ cd swh-environment - $ make docs-apidoc + $ make docs-assets This will *not* build the documentation in each module (there is `make docs` -for that), but will use `sphinx-apidoc` to generate documentation indexes for -each (sub)modules in the various Software Heritage components. +for that). -As `sphinx-apidoc` refuses to overwrite old documents, before proceeding you -might need to clean up old cruft with: - $ cd swh-environment - $ make docs-clean +### 2. Build the api docs for all swh python packages + $ cd swh-docs/docs + $ make apidoc -### 2. Build the documentation +### 3. Build the documentation $ cd swh-docs/docs $ make @@ -76,11 +87,13 @@ Publishing the doc ------------------ - $ cd docs - $ make install - $ xdg-open https://docs.softwareheritage.org/devel/ +The publication of the documentation is now managed by the [CI][7]. + + -For the above to work you need to have ssh access into the machine -hosting (currently `pergamon`), and write -access do the document root directory of that virtual host (currently granted -to all members of the `swhdev` UNIX group on Software Heritage machines). +[1]: http://www.sphinx-doc.org/ +[2]: https://forge.softwareheritage.org/source/swh-environment/ +[3]: https://graphviz.org +[4]: http://plantuml.com +[5]: https://tox.readthedocs.io/ +[6]: https://github.com/jd/pifpaf diff --git a/docs/Makefile b/docs/Makefile --- a/docs/Makefile +++ b/docs/Makefile @@ -3,17 +3,23 @@ SOURCEDIR = . BUILDDIR = _build HTMLDIR = $(BUILDDIR)/html +SWHPKGDIR = `python3 -c 'import swh; print(swh.__path__[0])'` INSTALL_HOST = pergamon.internal.softwareheritage.org INSTALL_DIR = /srv/softwareheritage/docs/webroot/devel INSTALL_GROUP = swhdev INSTALL_PERMS = g+rwX -html: fix-indices-stamp sphinx/html +SPHINXAPIDOC = sphinx-apidoc +APIDOC_DIR = apidoc +APIDOC_OPTS = --ext-viewcode +APIDOC_EXCLUDES = */tests */tests/* */*/tests/* */*/*/tests/* +APIDOC_EXCLUDES += */migrations */migrations/* */*/migrations/* */*/*/migrations/* +APIDOC_SWH_EXCLUDES = $(patsubst %,$(SWHPKGDIR)/%,$(APIDOC_EXCLUDES)) -fix-indices-stamp: sphinx/html - bin/copy-and-fix-subprojects-indices - touch $@ +apidoc_dep = apidoc-stamp + +html: sphinx/html sphinx/html: links-stamp apidoc-stamp images-stamp rec-build-stamp @@ -21,8 +27,9 @@ bin/ln-sphinx-subprojects touch $@ +apidoc: $(apidoc_dep) apidoc-stamp: - $(MAKE) -C ../../ docs-apidoc + $(SPHINXAPIDOC) $(APIDOC_OPTS) -o $(APIDOC_DIR) $(SWHPKGDIR) $(APIDOC_SWH_EXCLUDES) touch $@ images-stamp: @@ -33,13 +40,14 @@ # non-sphinx managed documentation artifacts (e.g., schema diagrams) are also # built. rec-build-stamp: $(wildcard ../../swh-*/docs/*.rst) - $(MAKE) -C ../../ docs + $(MAKE) -C ../../ docs-assets touch $@ clean: sphinx/clean bin/ln-sphinx-subprojects --remove $(MAKE) -C images clean rm -f *-stamp + rm -f $(APIDOC_DIR)/* distclean: clean make -C ../../ docs-clean diff --git a/requirements-swh-dev.txt b/requirements-swh-dev.txt new file mode 100644 --- /dev/null +++ b/requirements-swh-dev.txt @@ -0,0 +1,22 @@ +# Add here internal Software Heritage dependencies, one per line. +../swh-core +../swh-model +../swh-objstorage[testing] +../swh-scheduler +../swh-storage[schemadata] +../swh-loader-core +../swh-lister +../swh-journal +../swh-vault +../swh-loader-dir +../swh-loader-tar +../swh-loader-pypi +../swh-loader-debian +../swh-loader-mercurial +../swh-loader-svn +../swh-loader-git +../swh-archiver +../swh-web +../swh-deposit +../swh-indexer +../swh-mirror-forge diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1 +1,22 @@ # Add here internal Software Heritage dependencies, one per line. +swh-core +swh-model +swh-objstorage[testing] +swh-scheduler +swh-storage[schemadata] +swh-loader-core +swh-lister +swh-journal +swh-vault +swh-loader-dir +swh-loader-tar +swh-loader-pypi +swh-loader-debian +swh-loader-mercurial +swh-loader-svn +swh-loader-git +swh-archiver +swh-web +swh-deposit +swh-indexer +swh-mirror-forge diff --git a/swh/docs/django_settings.py b/swh/docs/django_settings.py new file mode 100644 --- /dev/null +++ b/swh/docs/django_settings.py @@ -0,0 +1,14 @@ +from swh.deposit.settings.development import * # noqa +import swh.web.settings.development as web + + +# merge some config variables +ns = globals() +for var in dir(web): + if var.isupper() and var in ns and isinstance(ns[var], list): + for elt in getattr(web, var): + if elt not in ns[var]: + ns[var].append(elt) + + +SECRET_KEY = 'change me' diff --git a/swh/docs/sphinx/conf.py b/swh/docs/sphinx/conf.py --- a/swh/docs/sphinx/conf.py +++ b/swh/docs/sphinx/conf.py @@ -20,6 +20,7 @@ 'sphinxcontrib.httpdomain', 'sphinx.ext.extlinks', 'sphinxcontrib.images', + 'sphinx.ext.viewcode', ] # Add any paths that contain templates here, relative to this directory. @@ -40,8 +41,9 @@ # A string of reStructuredText that will be included at the beginning of every # source file that is read. +# A bit hackish but should work both for each swh package and the whole swh-doc rst_prolog = ''' -.. include:: /swh_substitutions +.. include:: /../../swh-docs/docs/swh_substitutions ''' # The version info for the project you're documenting, acts as replacement for @@ -123,22 +125,15 @@ autodoc_member_order = 'bysource' autodoc_mock_imports = ['rados'] +modindex_common_prefix = ['swh.'] + # for the extlinks extension, sub-projects should fill that dict extlinks = {} # hack to set the adequate django settings when building global swh doc -# to avoid build errors -def source_read_handler(app, docname, source): - if 'swh-deposit' in docname: - os.environ.setdefault('DJANGO_SETTINGS_MODULE', - 'swh.deposit.settings.development') - django.setup() - elif 'swh-web' in docname: - os.environ.setdefault('DJANGO_SETTINGS_MODULE', - 'swh.web.settings.development') - django.setup() - - +# to avoid autodoc build errors def setup(app): - app.connect('source-read', source_read_handler) + os.environ.setdefault('DJANGO_SETTINGS_MODULE', + 'swh.docs.django_settings') + django.setup() diff --git a/tox.ini b/tox.ini new file mode 100644 --- /dev/null +++ b/tox.ini @@ -0,0 +1,28 @@ +[tox] +envlist=flake8 + +[testenv] +basepython = python3 + +[testenv:sphinx] +deps = + django < 2 + .[testing] + pifpaf +commands = + {envpython} -m pifpaf run postgresql -- make -C docs html + +[testenv:sphinx-dev] +deps = + django < 2 + -rrequirements-swh-dev.txt + pifpaf +commands = + {envpython} -m pifpaf run postgresql -- make -C docs html + +[testenv:flake8] +skip_install = true +deps = + flake8 +commands = + {envpython} -m flake8