diff --git a/conf/lister.yml b/conf/lister.yml --- a/conf/lister.yml +++ b/conf/lister.yml @@ -17,12 +17,14 @@ task_broker: amqp://guest:guest@amqp// task_modules: - swh.lister.bitbucket.tasks + - swh.lister.cgit.tasks - swh.lister.cran.tasks - swh.lister.debian.tasks - swh.lister.github.tasks - swh.lister.gitlab.tasks - swh.lister.gnu.tasks - swh.lister.npm.tasks + - swh.lister.packagist.tasks - swh.lister.phabricator.tasks - swh.lister.pypi.tasks task_queues: @@ -30,6 +32,8 @@ - swh.lister.bitbucket.tasks.IncrementalBitBucketLister - swh.lister.bitbucket.tasks.RangeBitBucketLister - swh.lister.bitbucket.tasks.ping + - swh.lister.cgit.tasks.CGitListerTask + - swh.lister.cgit.tasks.ping - swh.lister.cran.tasks.CRANListerTask - swh.lister.cran.tasks.ping - swh.lister.debian.tasks.DebianListerTask @@ -47,6 +51,8 @@ - swh.lister.npm.tasks.NpmIncrementalListerTask - swh.lister.npm.tasks.NpmListerTask - swh.lister.npm.tasks.ping + - swh.lister.packagist.tasks.PackagistListerTask + - swh.lister.packagist.tasks.ping - swh.lister.phabricator.tasks.FullPhabricatorLister - swh.lister.phabricator.tasks.IncrementalPhabricatorLister - swh.lister.phabricator.tasks.ping diff --git a/q b/q new file mode 100644 --- /dev/null +++ b/q @@ -0,0 +1,1662 @@ +commit 387ad65d072e7f612a20b365de5ab386c668b442 (HEAD -> master) +Author: Antoine Lambert +Date: Mon Sep 9 17:24:00 2019 +0200 + + swh-listers-worker/entrypoint: Update listers init commands + + Synchronize with latest changes in swh-lister component. + +commit 16d51553507cfff08258ed4fd5c98a3ec6e585b3 +Author: Antoine Lambert +Date: Mon Sep 9 17:18:46 2019 +0200 + + entrypoints: Stop using deprecated swh-db-init command + +commit e6fd2088ba8f210ddcb91d7f1393f08a4a815612 +Author: Antoine Lambert +Date: Mon Sep 9 17:17:28 2019 +0200 + + README.md: Rename api_baseurl lister task argument to url + + Synchronize with changes from commit rDLSb810876ef827afcdd7b00273dc8aaa13a8c6c2d2. + +commit 44e1467ca2fc0dcd5865d3de1b8288e6b5fa22d6 (origin/master, origin/HEAD) +Author: Valentin Lorentz +Date: Mon Aug 12 16:10:20 2019 +0200 + + Escape $() in the 'cat << EOF' command. + +commit 307ffd6a3783f8e227b16f1d1fa6b7d8682c4401 +Author: Antoine Lambert +Date: Mon Aug 5 13:05:27 2019 +0200 + + swh-web/entrypoint: Improve admin user creation script + +commit ba4331941b51654c9323f180cc151e46a25876f0 +Author: Antoine Lambert +Date: Tue Jul 30 17:33:00 2019 +0200 + + Fix swh-vault service by removing calls to deprecated cli + +commit 675bd91f5f06e6f6a5cc1d92d2bb47b0a7a1f464 +Author: Valentin Lorentz +Date: Wed Jul 24 12:05:36 2019 +0200 + + Update throttle scope config with new required scope. + +commit bb7dfc599820292c1048fc80eb91ea5aef46793f +Author: Valentin Lorentz +Date: Wed Jul 24 12:05:03 2019 +0200 + + remove absolute path + +commit fb347f42b101c1f11d3ff96d16c3f0da64ecb2ce +Author: Valentin Lorentz +Date: Tue Jul 23 18:53:52 2019 +0200 + + Add Prometheus JMX exporter for Cassandra. + +commit 4f6be134cfa1978b9c8ee0887e627f57f49254fb +Author: Valentin Lorentz +Date: Tue Jul 23 18:50:24 2019 +0200 + + Add an option to run swh-storage using the Cassandra backend. + +commit 1b0a2c0442400f87648235da6469be7d08e8513f +Author: Valentin Lorentz +Date: Tue Jul 23 18:49:07 2019 +0200 + + Add 'cassandra-seed' and 'cassandra' containers + +commit 2edb4967a9f70bb815a16238424504b5e01a01d6 +Author: Stefano Zacchiroli +Date: Thu Jul 11 16:29:32 2019 +0200 + + add code of conduct document + +commit d9a926b3791b984e70665fe4ac994ac024cd67e0 +Author: Valentin Lorentz +Date: Wed Jul 10 16:27:33 2019 +0200 + + Update indexer-journal-client config and entrypoint. + +commit c21a4776e4258c0294c4bd4733d598f78712d179 +Author: Antoine Lambert +Date: Tue Jul 2 13:50:13 2019 +0200 + + tests: Avoid failure when a service exited with 0 + +commit 4795b8ca3b7132c730fb4bb6cfa9eab40bd04050 +Author: Antoine R. Dumont (@ardumont) +Date: Fri Jun 28 18:01:30 2019 +0200 + + Add Archit Agrawal as contributor + +commit 9b12218d59cd5aa2e1feaa73783a835d5c983efd +Author: David Douard +Date: Thu Jun 27 10:38:28 2019 +0200 + + entrypoints: ensure gunicorn services can handle more than one connection + +commit 560df3bc13c8853b94e646a00177ccd9c6784ada +Author: Archit Agrawal +Date: Wed Jun 19 14:18:22 2019 +0530 + + README: Update docker-compose logs command + + docker-compose logs command attached swh-lister-debian + container in readme, whereas now all the lister are in + single container swh-lister. + +commit 9be73e3693c3a295b690e09025382a94a20074b1 +Author: Antoine Lambert +Date: Tue Jun 18 17:00:57 2019 +0200 + + conf/web: Remove now useless ReCAPTCHA configuration + + Related T1815 + +commit dee708ec4a79dc6a7a2db4788d7ea2d40c0a309d +Merge: 490495d 179ce32 +Author: Archit Agrawal +Date: Thu Jun 13 11:04:10 2019 +0200 + + Add CRAN, GNU, and phabricator lister in conf/lister.yml + + Reviewers: #reviewers, ardumont + + Reviewed By: #reviewers, ardumont + + Differential Revision: https://forge.softwareheritage.org/D1575 + +commit 179ce3202a58abeb3c88c8166169045265c41e3f +Author: Archit Agrawal +Date: Thu Jun 13 11:04:10 2019 +0200 + + Add CRAN, GNU, and phabricator lister in conf/lister.yml + + Reviewers: #reviewers, ardumont + + Reviewed By: #reviewers, ardumont + + Differential Revision: https://forge.softwareheritage.org/D1575 + +commit 490495d31ced048f45845a234a1756bc85afd378 (new-scheduler-task-names) +Author: Antoine Lambert +Date: Mon May 20 15:35:30 2019 +0200 + + Update scheduler task names to new ones + + Related T1508 + +commit 022463d176a07519e9dfc2ffddd90c76cae0c064 +Author: Antoine Lambert +Date: Tue May 21 13:21:15 2019 +0200 + + Dockerfile: Ensure all Python dependencies get installed + + Related T1678 + +commit 62d904dcd60e68734c6cd5aa7488487540ed3a68 +Author: Antoine Lambert +Date: Wed May 15 16:52:34 2019 +0200 + + Use new 'swh scheduler' cli instead of deprecated swh-scheduler + + Related T1671 + +commit 8d48959f9643aa345049668fee8461a66a27c5c3 +Author: David Douard +Date: Wed May 15 11:41:09 2019 +0200 + + Dockerfile: update to python 3.7 + +commit 00d3f8f358403b86bbc8856020ad2524341c58b1 +Author: David Douard +Date: Wed May 15 11:41:46 2019 +0200 + + README: update doc to use the new 'swh' master cli tool + +commit ac814596ba03d925d53d8a1726af3271a0588b6d +Author: Antoine Lambert +Date: Tue May 14 17:57:11 2019 +0200 + + tests: Set no-ansi docker-compose option when stdout is not a tty + +commit c76178577671e9820d9768cf9946a442ad127c3c +Author: Antoine Lambert +Date: Fri May 10 15:29:18 2019 +0200 + + tests: Also dump logs when a service died unexpectedly + +commit 523c08ba9d1a5bbaf300ae1e31a18955a6e7fd72 +Author: Antoine Lambert +Date: Fri May 10 15:16:50 2019 +0200 + + tests: Print colored output only if stdout is a tty + +commit bf9bac2e47961899a4f587d0d5e3cca0457001ce +Author: Antoine Lambert +Date: Fri May 10 14:09:00 2019 +0200 + + tests: Fix 'input device is not a TTY' on Jenkins + +commit 5c9b912b4bc46c6c971b13e54b32355475585070 (docker-compose-integration-tests) +Author: Antoine Lambert +Date: Fri Apr 26 17:12:41 2019 +0200 + + tests: Add integration tests for the Software Heritage stack + + Closes T1618 + +commit 9ac891b8914bf90006e2b0c222d71bf8b3b15282 +Author: Antoine Lambert +Date: Fri Apr 26 14:28:01 2019 +0200 + + services/zookeeper: Set restart policy to always + + It exists cases where zookeeper will crash at startup due to a race condition related + to the reading of a configuration file. So ensure it will be automatically restarted + when a such a case appears. + + Related T1618 + +commit 875c50aa3920350da43d4cfd946e0658f14ced77 +Author: Antoine Lambert +Date: Wed Apr 24 16:43:41 2019 +0200 + + conf/web.yml: Disable swh web api rate limiting + + Related T1618 + +commit 848401f512e666855f421507e319d6f2a709ed13 +Author: Antoine R. Dumont (@ardumont) +Date: Sat Apr 13 16:15:53 2019 +0200 + + README: Update to demonstrate how to run the backfiller + +commit 0bcb82bcd45748f2aae5a4532a6f2674af01e94c +Author: Antoine R. Dumont (@ardumont) +Date: Thu Apr 4 14:52:15 2019 +0200 + + docker-compose: Add journal-backfiller + +commit c04db2b6cf7f923bad3c7ac39514ea51130457e8 +Author: Antoine R. Dumont (@ardumont) +Date: Sat Apr 13 15:59:22 2019 +0200 + + Ignore override docker-compose file + +commit 905d3fd79f8a5bf88a7210f7f3f9542fda3e0168 +Author: Antoine R. Dumont (@ardumont) +Date: Fri Apr 12 11:49:20 2019 +0200 + + conf/deposit: Ensure swh-deposit is part of the allowed_host setting + +commit d6981f3c820ffaa5005aba45a5ac659b3972806b +Author: Antoine R. Dumont (@ardumont) +Date: Fri Apr 12 10:24:45 2019 +0200 + + swh-deposit/entrypoint: Initialize the needed test user + +commit f68bb33b1493d7ef46471e3162abd03e6d6b0021 +Author: Antoine R. Dumont (@ardumont) +Date: Fri Apr 12 11:47:44 2019 +0200 + + conf/loader: Add loader/checker worker tasks + +commit 77f4b2e0617be57282e5ab4a972f7a643768e668 +Author: Antoine R. Dumont (@ardumont) +Date: Fri Apr 12 11:48:27 2019 +0200 + + conf/deposit: Fix misconfiguration + + swh-scheduler does not exist + +commit 2f87f477046fc82b7ce4b6fede712c051e943c14 +Author: Antoine R. Dumont (@ardumont) +Date: Fri Apr 12 11:49:56 2019 +0200 + + conf/loader: Add missing deposit loader/checker setting + + This will need to be reworked on the deposit's side to ensure names + are explicit. + +commit ca58318d9a24ef66d9c776a8d8fe977e9d69c0e2 (add-npm-loader-conf) +Author: Antoine Lambert +Date: Thu Apr 11 10:55:52 2019 +0200 + + Add npm loader and its configuration + + Related T1631 + +commit 928075ce2069d95571a4b1508bf487c07957b16b (swh-web-db-conf) +Author: Antoine Lambert +Date: Wed Apr 10 17:21:29 2019 +0200 + + services/swh-web: Specify database file through configuration + +commit a8f631fcb63ab9a69a165ded7a2dc7aed4b2f5d4 +Author: Antoine Lambert +Date: Wed Apr 10 17:09:43 2019 +0200 + + README.md: Fix outdated swh-scheduler command invocation + +commit 44b9e7978b518484531eef70a8682cd217035232 +Author: Antoine Lambert +Date: Wed Apr 10 15:28:23 2019 +0200 + + conf/storage.yml: Fix configuration layout + +commit 30e9c934ef206e3f3711f19912964a14142736b6 +Author: David Douard +Date: Wed Apr 10 11:00:05 2019 +0200 + + lister: allways run the db initialization step + +commit 6d49580cd56ce9411a88065310430f2b773c834c +Author: David Douard +Date: Wed Apr 10 10:59:24 2019 +0200 + + replica: ensure entrypoints are defined in replica related services + +commit b7f02d8add7f656904d43553c53ad5d3a36c974a +Author: David Douard +Date: Wed Apr 10 11:49:33 2019 +0200 + + dockerfile: install python packages in a venv under the swh user + + this should prevent some caveats with packages installed from sources. + +commit 53fa588c7fc22a19fa166fc1c541f276be8f6e93 +Author: David Douard +Date: Tue Apr 9 15:18:39 2019 +0200 + + Factorise the pip install of source repos in a shell function + +commit 6558c6c8ad1811ac0d75c7a611bba78a73639f5c +Author: David Douard +Date: Wed Apr 10 11:23:29 2019 +0200 + + entrypoint: pgsql has been moved to ~swh/ + +commit 7b0ea28036065ea2a07542435a34e7cc5afefb44 +Author: David Douard +Date: Mon Apr 8 13:20:58 2019 +0200 + + Remove entrypoint.sh files from the Dockerfile + + compose: ensure entrypoint is defined everywhere and make services more consistent + + - ensure build/image is allways defined in the same order, + - typo in swh-lister + +commit d1f5b87ba8c7d467523f854054c9a63be1ff5e39 +Author: David Douard +Date: Mon Apr 8 12:11:47 2019 +0200 + + Dockerfile: small readability refactoring + +commit 9359da6dbf950d4084d127a2df03f5717e2a1623 +Author: David Douard +Date: Mon Apr 8 11:56:00 2019 +0200 + + Add support for a kafka-powered replica of the graph. + + This is in a dedicated compose file so that one must choose to run it (i.e. + it is not a service started by default when using a 'docker-compose up'). + + It: + + - overrides the swh-storage service to activate the kafka direct writer + on swh.journal.objects prefixed topics using thw swh.storage.master ID, + - overrides the swh-web service to make it use the replica instead of the + master storage, + - starts a db for the replica, + - starts a storage service based on this db, + - starts a replayer service that runs the process that listen to kafka to + keeps the replica in sync. + +commit 587f9df57a8058d7fd8b6cbeb2571ba130a804e2 +Author: David Douard +Date: Tue Apr 9 15:22:34 2019 +0200 + + lister: Use explicit postgresql URLs as a workaround for a sqlalchemy issue + + see. https://github.com/sqlalchemy/sqlalchemy/issues/4601 + +commit 20b0733b84f30d67ec93393cc7c2455af4eb5ac8 +Author: David Douard +Date: Tue Apr 9 15:02:38 2019 +0200 + + swh-web: enforce the database file to be created in either $HOME or /tmp + + but NOT in the swh-web python package. + +commit 8a2458ee2bf06af4bb89e91daea48a9467a93f70 +Author: David Douard +Date: Wed Apr 10 12:06:00 2019 +0200 + + compose: fix inconsistent dependency + + swh-storage-api does not exists. + +commit adeb8787f629e7ab391b1d02ad4b3a619c2ed656 +Author: Antoine R. Dumont (@ardumont) +Date: Tue Apr 9 13:13:36 2019 +0200 + + docker-compose: Remove journal client sample + +commit a1beb091c9ec691e654d7147089275e9bf61ef61 +Author: Antoine R. Dumont (@ardumont) +Date: Tue Apr 9 13:11:38 2019 +0200 + + docker-compose: Remove storage listener and journal publisher + +commit abc5a2d99f2d3dd1537667abed22997fedc029fd (update-kafka-config) +Author: Antoine Lambert +Date: Fri Apr 5 18:15:57 2019 +0200 + + Update kafka based services configuration + +commit 669cec1a7026542f504371cbf72e7d0da3315e3d (kafka-manager) +Author: Antoine Lambert +Date: Fri Apr 5 13:29:28 2019 +0200 + + docker-compose.yml: Add kafka-manager service + +commit 2eb1be67bd30b5ef3db63dc0fc6ad0421c22e1b1 +Author: Antoine Lambert +Date: Thu Apr 4 17:58:13 2019 +0200 + + Dockerfile: psycopg2-binary is no more needed since psycopg2 2.8 release + +commit d7a53b16b822d7ce578e93fd1156219e2dea1970 +Author: Antoine Lambert +Date: Wed Apr 3 19:42:19 2019 +0200 + + README.md: Fix typos + +commit c4d465b275161b7b1960cb85b86e2f77153e93cf +Author: Antoine Lambert +Date: Wed Apr 3 14:34:17 2019 +0200 + + services/swh-web: Automatically create an admin account + +commit 0a261d233f0db40dba06a2431a2b40914e65a480 +Author: Antoine Lambert +Date: Wed Apr 3 14:33:04 2019 +0200 + + services/swh-storage-listener: Fix pgsql setup + +commit 23e497ca46fa1abdd154e4a8c7b02781d4b6cfa4 (README-update) +Author: Antoine Lambert +Date: Wed Apr 3 11:13:27 2019 +0200 + + README.md: Document how to rebuild the swh/stack docker image + + Also fix a typo and some remarkup rendering issues. + +commit 9d7051c59eaaab380489033e5c2e4a2f0b13909c +Author: Antoine Lambert +Date: Fri Mar 29 14:26:27 2019 +0100 + + Remove file committed by mistake + +commit 90d0be8359bb1820de942baeff727a59025c6370 +Author: Antoine Lambert +Date: Fri Mar 29 13:57:32 2019 +0100 + + env/indexers-db.env: Fix indexers database name + +commit a9bd7f3ec4e98b3896f673e74a859f65d3147e8e +Author: Antoine R. Dumont (@ardumont) +Date: Wed Mar 27 14:58:48 2019 +0100 + + Dockerfile: Install ngrep as default tools on node + + That allows to check for stats information: + + For example: + ``` + docker-compose exec swh-storage bash -c 'ngrep -d lo udp and port 8125' + ``` + +commit f6791cce6ac1d4540ca50c3ee64bfb1d5829dd4b (harmonize_pgsql_config) +Author: Antoine Lambert +Date: Wed Mar 27 13:49:21 2019 +0100 + + Harmonize and factorize PGSQL configuration + +commit c5a48242d23b80aef24ad2fca5d719a47e223ede +Author: Antoine R. Dumont (@ardumont) +Date: Thu Mar 14 14:39:23 2019 +0100 + + journal-publisher: Update the journal publisher entrypoint + + As the configuration is no longer implicit, add the missing bits and + pieces. + + Related D1239 + Related T1276 + +commit 7c26fe6e63b801e140db79752e2f41a0d5fbb2db (swh-stack-image) +Author: Antoine Lambert +Date: Wed Mar 13 11:26:02 2019 +0100 + + Use a single docker image instead of one per swh service + + In order to speed up the docker-compose build process but also to remove + duplicated instructions in dockerfiles, prefer to use a single docker image + containing the full swh stack. Each swh service will run upon it while their + specific configuration (for instance entrypoint) will be picked from the + docker-compose.yml file. + +commit 11fbcde86e45553bbdc5d515816c1c4d53f7f858 +Author: Antoine R. Dumont (@ardumont) +Date: Mon Mar 11 14:52:39 2019 +0100 + + README: Fix typos and sentence phrasing + +commit 4bb6811b0a3e783b5c1464bcc5a49dec97a48c4c (db-wait-for-it) +Author: Antoine Lambert +Date: Mon Mar 11 12:00:31 2019 +0100 + + dockerfiles: Use wait-for-it script to ensure postresql dbs are up + +commit 6417e87e38e09f92e0a40e1d36073ff6848affb7 +Author: David Douard +Date: Fri Mar 8 13:12:43 2019 +0100 + + readme: make it compliant with phabricator's limited remarkup format + +commit c566781302293d0e98cdfc50c4c1cebc9956a026 +Author: David Douard +Date: Tue Mar 5 15:31:10 2019 +0100 + + readme: integrate the docker-based development setup from the main's doc + + and update several cmdline examples. + +commit 8871e9cd32eb5f9fb4b92d949258a408daf51709 +Author: David Douard +Date: Tue Mar 5 15:26:22 2019 +0100 + + swh-scheduler: add SWH_CONFIG_FILENAME and SWH_SCHEDULER_CONFIG_FILE env vars + + in all the swh-scheduler related services. Also configure the + CELERY_BROKER_URL env var. + + These allow to use the swh-scheduler directly in the container without + any cmdline option, as well as the celery command. + +commit d192c1cde3d7106002c397bb6a7cac0fecfe77d3 +Author: David Douard +Date: Tue Mar 5 15:24:56 2019 +0100 + + docker-compose: move the swh-indexer-journal-client in the journal "section" + +commit aecd6074c01dab7a3cc17d4d62ea9e8a181f5667 (swh-web-install-pandonc) +Author: Antoine Lambert +Date: Thu Mar 7 17:48:25 2019 +0100 + + dockerfiles/swh-web: Install pandoc used to render README.rst file to HTML + +commit 9862f75ac849fb8963e9f0a15e7a4db62a00f0e6 (fix-storage-gunicorn-timeout) +Author: Antoine Lambert +Date: Thu Mar 7 16:16:46 2019 +0100 + + dockerfiles/*/entrypoint.sh: Use same gunicorn timeout value as in production + +commit 5830c7a7d2890db9faf3c3e6cf035d2bb88c716a (wait-kafka-server) +Author: Antoine Lambert +Date: Thu Mar 7 14:47:24 2019 +0100 + + dockerfiles: Ensure kafka server is up before starting related services + +commit 19dd6fd747fb3c480806b942878db4639d6942be (ensure-amqp-server-up) +Author: Antoine Lambert +Date: Thu Mar 7 11:45:45 2019 +0100 + + dockerfiles: Ensure RabbitMQ server is up before starting related services + +commit 9f8ea5e24591fb3fed7455a99cc4440eaee216e2 (update-objstorage-conf) +Author: Antoine Lambert +Date: Wed Mar 6 16:29:33 2019 +0100 + + conf/objstorage: Use a higher value for client_max_size + + This value is the default one that was previously used in objstorage + implicit configuration and make loaders happy + +commit 8706815679b74be63e9079777e4f6629661629c5 (fix-vault-db) +Author: Antoine Lambert +Date: Wed Mar 6 11:38:20 2019 +0100 + + vault: Fix database configuration + +commit f7f358414f6d17b4caf47b4387a46d35899e7553 +Author: Stefano Zacchiroli +Date: Mon Mar 4 14:09:58 2019 +0100 + + objstorage conf: increase max payload size + +commit 0b031ba074dff7ec8f100f5f482e29b6bfed26d8 +Author: David Douard +Date: Mon Mar 4 13:47:09 2019 +0100 + + web: remove djangosettings.py file and use swh.web.settings.development + +commit 7b3213293ca1670a738d540cbba05e87e5cf6042 +Author: David Douard +Date: Mon Mar 4 13:46:19 2019 +0100 + + conf: make swh-web always serve assets + +commit 0c89cd3350bbd871d7056937055bf17d7b67e72a +Author: David Douard +Date: Mon Mar 4 13:45:00 2019 +0100 + + dockerfile: ensure swh-web's version of pip is up to date + +commit aac4cff40773c735eec9f5eeff5d4643b1fe2e28 +Author: David Douard +Date: Mon Feb 25 17:53:13 2019 +0100 + + grafana: replace the tasks dashboard with olasd's task-processing one + +commit 5e8af88dc7f85e509af7b1b973fc11401fcd620c +Author: Nicolas Dandrimont +Date: Tue Feb 19 11:45:33 2019 +0100 + + Revert "Install kombu from sources in scheduler-worker as a workaround for T1491" + + This reverts commit f8ca643a0508f9d41dea87733d9fca5a2f883dc9. + +commit 1f10d774be7bcc653b4dc7530675b0c681c57290 +Author: David Douard +Date: Thu Feb 21 18:29:41 2019 +0100 + + Fix celery workers conf files: explicit list of queues is now required + +commit 777d5be54ec079928ddafe7e73d40f4cf4b168b5 +Author: David Douard +Date: Thu Feb 21 18:28:05 2019 +0100 + + Fix nginx revert proxy for rabbitmq + + make it not normalize the URI passed to the backend. + +commit 48038bdb707a1ba38e5d2e0afa87c3c22248334a +Author: David Douard +Date: Thu Feb 21 12:04:36 2019 +0100 + + Upgrade to pg 11 + + Note that this will kill your existing db containers... + +commit 3a7e4df76175d89ce0df7fc8fbcd46c3f571e93b +Author: David Douard +Date: Wed Feb 20 17:30:57 2019 +0100 + + Activate a 10s heartbeat for celery workers + + makes it easier to get a consistent celery status. + +commit e5dc74ec4e4742df64e938a5c0ba5bb4ddcf9629 +Author: Antoine R. Dumont (@ardumont) +Date: Sat Feb 23 01:48:21 2019 +0100 + + docker-compose: Remove unused command + +commit a8429eba9a1064a66746f3d47bafd67e00c1c554 +Author: Antoine R. Dumont (@ardumont) +Date: Sat Feb 23 01:16:44 2019 +0100 + + swh-web: Unify production and development start up + + Related T1533 + +commit 8bec6a01cdb026c115f9dfa68d4603a02cc22e59 +Author: Antoine R. Dumont (@ardumont) +Date: Sat Feb 23 01:06:26 2019 +0100 + + indexer-storage: Remove expose 5002 instruction + +commit a19e47374717d0430ce2faa419a10bcdfe8d03dd +Author: Antoine R. Dumont (@ardumont) +Date: Sat Feb 23 00:41:12 2019 +0100 + + swh-scheduler: Unify production and development start up + + Related T1533 + +commit 3d1d2dc3f43204b66941277d66949ecab96405ab +Author: Antoine R. Dumont (@ardumont) +Date: Thu Feb 21 20:01:30 2019 +0100 + + swh-objstorage: Unify production and development start up + + Related T1533 + +commit 6d9720b41fcbdcaffe332aae571b29fc8ab1d964 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Feb 21 14:47:47 2019 +0100 + + swh-idx-storage: Unify production and development start up + + Related T1533 + Related D1170 + +commit 7894567e149bd7c4caafcfbffac976a88d0cffb5 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Feb 21 12:27:12 2019 +0100 + + swh-storage: Unify production and development start up + + This now: + - load configuration form the SWH_CONFIG_FILENAME + - uses gunicorn (same as in production) + + Depends on D1167 + Related T1533 + +commit 75dcaa82059cc2338c94f14ed7d383a41c9361fd +Author: Antoine R. Dumont (@ardumont) +Date: Wed Feb 20 13:40:10 2019 +0100 + + deposit.env: Add missing required django settings module + +commit 898e2f4563e6b2438ce6758bdc35c3612cc7afca +Author: Antoine R. Dumont (@ardumont) +Date: Wed Feb 20 13:28:02 2019 +0100 + + swh-deposit: Use proper configuration scheme + + Adapted according to latest diff. + +commit 79ef8f347f56db3c77d075f6057b521928969f03 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Feb 20 13:29:44 2019 +0100 + + swh-deposit: Remove not yet used configuration + +commit 2ed0c7833c5b4d2feb8af7ffcd029443bde0565b +Author: David Douard +Date: Tue Feb 12 12:24:32 2019 +0100 + + Add the vault services (api and worker) and make them avaible in swh-web + +commit bb38cd93622ef5c40af7bdcdb55349c0f2060ce8 +Author: David Douard +Date: Tue Feb 12 12:17:50 2019 +0100 + + Deploy a djangossettings.py for swh-web that allows to set a dumd asmin password + + It should be a matter of either: + + docker-compose exec swh-web bash + root@183ba8e88fd6:/tmp/swh# python -m swh.web.manage createsuperuser \ + --settings djangosettings --username admin --email noone@is.inocent + + or: + + docker-compose run swh-web createsuperuser --settings \ + djangosettings --username admin --email noone@is.inocent + +commit 0ac8db330055f132ffc7e3589b60d803cbf39856 +Author: David Douard +Date: Tue Feb 12 12:14:22 2019 +0100 + + Move options and arguments of the swh manage command as COMMAND + +commit 452185dac5ef604bd838d16ee13c1b1826a4d90c +Author: David Douard +Date: Wed Feb 6 18:24:10 2019 +0100 + + Very beginning of a grafana service + +commit 7000265592f091254ff49023cdd15cc78eea78ff +Author: David Douard +Date: Wed Feb 6 18:22:28 2019 +0100 + + Update the lister entrypoint to make latest cli updates + + no more --create-tables and --with-data options. + +commit 7a0178e5c8392010ebd8dc0f5dbcc4b70ef870b4 +Author: David Douard +Date: Wed Feb 6 10:15:35 2019 +0100 + + Improve a bit the README file + + - add a warning about possible disk gavage, + - remove reference to flower, + - add a note on possible caveats when using local src repos in the docker. + +commit 842f819f3439a82db6c35841dae9acc6c35be68f +Author: David Douard +Date: Wed Feb 6 10:13:06 2019 +0100 + + Merge all loaders and listers services in swh-loader and swh-lister + + in order to simplify the docker environment. + + Also comment the flower service, since it's pretty unreliable for now. + +commit 29c9481102d33d7c3c917028927a5846e03addaa +Author: David Douard +Date: Wed Feb 6 10:10:30 2019 +0100 + + Ensure PATH contains ~/.local/bin for dockers where 'pip install --user' is used + +commit f8ca643a0508f9d41dea87733d9fca5a2f883dc9 +Author: David Douard +Date: Wed Feb 6 10:04:14 2019 +0100 + + Install kombu from sources in scheduler-worker as a workaround for T1491 + +commit 7ddf391cdccbbf864a7922088e611043087abf3b +Author: David Douard +Date: Wed Feb 6 09:57:14 2019 +0100 + + Reduce verbosity of the journal pusblisher and storage listener. + +commit a1d4f5a8f5de39f83759dbb6232acedbe2df7df3 +Author: David Douard +Date: Thu Jan 31 17:59:33 2019 +0100 + + Move all swh config file in conf/ and use volumes to inject them in dockers + + so that there is no need to rebuild the image when modifying one of them. + +commit 1b6edf887e772f72553194706f4fd2ed9546100e +Author: David Douard +Date: Thu Jan 31 17:34:00 2019 +0100 + + Move conf and env files in dedicated directories + +commit 144f580b466264ed24e1cc4e1c15938927fd8cbe +Author: David Douard +Date: Thu Jan 31 16:59:23 2019 +0100 + + Workers' celery config is now managed as the [celery] section of the main config file + +commit 4dcbc2cf5e5aae7b2a2bb9598ddda6ef9748a5d5 +Author: David Douard +Date: Thu Jan 31 16:57:00 2019 +0100 + + Update config system for scheduler services + +commit f1aae8764ed07a7b7f82eedf0104f660708c75dd +Author: David Douard +Date: Thu Jan 31 16:54:39 2019 +0100 + + Reduce the slicing for objstorage + + it's probably best in the context of adocker test env... + +commit aed3cf803d1294808c91642571ae0739fe5afa02 +Author: David Douard +Date: Thu Jan 31 16:53:42 2019 +0100 + + Fix a typo in indexer's Dockerfile + +commit c90fb7242ff6f86a48e6fc87f6937ad2be0a063a +Author: David Douard +Date: Thu Jan 31 16:52:56 2019 +0100 + + Ensure pip, setuptools and wheel are up to date at image build time + +commit 7f8ff5e21523e282e0e28e4616c09a0777b1d362 +Author: David Douard +Date: Thu Jan 31 16:49:58 2019 +0100 + + Add a prefix in celery workers' hostnames to prevent confusion eg. with pypi + + for which both the lister and the loader are named just 'pypi'. + +commit 6f6b92cdb1731196bf4686f37e15433e357cea09 +Author: David Douard +Date: Thu Jan 31 16:46:12 2019 +0100 + + Small fix for the prometheus service + + looks something has changed recently in the prom docker, not sure + how to know what exactly, it's unclear to ne how the + prom/prometheus published on docker's hub is actually built. + +commit 9be3d5233674686e68eb45fa9ac6e1522456e6c2 +Author: David Douard +Date: Wed Jan 30 13:52:50 2019 +0100 + + Update config of the scheduler + + adapt to latest swh-scheduler's refactorings. + +commit 652f9f2709e27d148fb5e8522b515a7bc07f6221 +Author: David Douard +Date: Wed Jan 30 13:52:14 2019 +0100 + + Fix the celery worker's hostname + +commit 4d33cbaa87b1c064bd6bd4a4d01778903d4320a7 +Author: David Douard +Date: Wed Jan 30 13:50:39 2019 +0100 + + Indexers have been refactored so we now only one celery worker + + capable of doing all the indexers' work. + +commit bb63e0aed9c6599f57bb07049ea2df68e1a15af9 (swh-web-conf-update) +Author: Antoine Lambert +Date: Fri Jan 25 19:40:55 2019 +0100 + + swh-web/web.yml: Disable recaptcha integration + + Closes T1497 + +commit a2778431afc150771960c061cad13b24757cbdd6 +Author: Nicolas Dandrimont +Date: Thu Jan 24 17:04:41 2019 +0100 + + Add a statsd-exporter instance + + The buckets for the histogram mapping are mostly arbitrary for now. + +commit 6df984762351a39bf73870ff093f446ad8b4b4d8 +Author: Nicolas Dandrimont +Date: Wed Jan 23 14:33:41 2019 +0100 + + Add a prometheus instance + + The prometheus instance is available through http://localhost:5080/prometheus. + + Also seed the prometheus configuration so it can find its own metrics. + +commit b3c973594ab094500cdbf9103d0a40623e434e62 +Author: Nicolas Dandrimont +Date: Thu Jan 24 16:58:40 2019 +0100 + + nginx: Add error logging at info level on stderr + +commit 8fa21cc9d1cf383da9979292ec7f8c9ebfb1be0c +Author: Nicolas Dandrimont +Date: Wed Jan 23 16:00:14 2019 +0100 + + Move nginx to port 5080 + + This avoids port mismatch issues when doing proxy rewriting + +commit b35153426aed954d5ee40dc1d195c540581cdb1f +Author: Nicolas Dandrimont +Date: Wed Jan 23 14:31:26 2019 +0100 + + Avoid nginx depending on all the web-facing containers + + To do so, we work around nginx's static DNS resolution of proxy upstreams by: + - making sure all proxy upstreams are set in variables + - hardcoding the DNS resolver (to the always-available internal Docker DNS + resolver) + + This makes nginx resolve the proxy upstreams at runtime rather than on start. + +commit cc26d77abd4a041838a9fdfe2b1b67b101be02e5 +Author: Nicolas Dandrimont +Date: Thu Jan 24 16:59:57 2019 +0100 + + Add a Makefile with a check-staged target to please the pre-commit hook + +commit e0795aa1d5e75d3f15e35576f7569a4cd8bbbe16 +Author: David Douard +Date: Thu Jan 10 13:45:37 2019 +0100 + + Rewrite the 'How to use' section as a 'Quick Start' + + with a step-by-step approach. + + Also add a section listing ports published on the host and a section on + tasks (SWH vs. Celery). + +commit b24f45b91d5fed377591cb6a2f43e9789ede32f0 +Author: David Douard +Date: Mon Jan 14 14:41:55 2019 +0100 + + Add an nginx service to group all http services on the same service endpoint + + namely: + + - http://localhost:5080/ -> web app + - http://localhost:5080/flower/ -> celery dashboard + - http://localhost:5080/rabbitmq/ -> amqp dashboard + - http://localhost:5080/scheduler/ -> swh-scheduler API + - http://localhost:5080/storage/ -> swh-storage API + - http://localhost:5080/indexer-storage/ -> swh-indexer-storage API + - http://localhost:5080/deposit/ -> swh-deposit API + - http://localhost:5080/objstorage/ -> swh-objstorage API + +commit 04d38c79e427cef48bdf14d07a1ec22b82df7b84 +Author: David Douard +Date: Thu Jan 10 13:44:20 2019 +0100 + + Try to rationalize a bit ports published on the host + + The idea is to keep all the ports for the swh platform in the 50XX range + and to try to prevent collision with standard ports (for services like + rabbitmq, flower, etc.) + +commit b0cf67760891657cd0a47285d395ae0658ceda8c +Author: David Douard +Date: Tue Jan 15 17:35:58 2019 +0100 + + Fix tab indent in dockerfiles/swh-indexer-worker/entrypoint.sh + +commit 4d1f1a3ef3b4a07955910906a73eb690706808b1 +Author: David Douard +Date: Mon Jan 14 12:33:27 2019 +0100 + + Rename celery workers as @ + + instead of celery@. + +commit e1d3e4241823ffb47a7c3d0a4f5ecc352f16ef3f +Author: David Douard +Date: Mon Jan 14 12:28:13 2019 +0100 + + Use full postgresql connexion URLs everywhere + +commit 3c3f86d3ef4d9d52da0c81859024c8cc2b9a5c3a +Author: David Douard +Date: Mon Jan 14 12:19:39 2019 +0100 + + Use Python 3.6 since celery 4.2 is not yet compatible with python 3.7 + + Mainly due tu https://github.com/celery/celery/issues/5101 + +commit ccf5a2579c05db0def8be43a249bbba085935241 +Author: Valentin Lorentz +Date: Mon Jan 14 16:10:30 2019 +0100 + + Add license indexer. + +commit fc9a631156f7e45b146ce06e9c8c7dbca0c315a2 +Author: Valentin Lorentz +Date: Fri Jan 11 12:05:23 2019 +0100 + + Make the indexer-journal-client container depend on the scheduler api. + +commit 99afd67ae3e045ba0e33bd0a7fcbd2e8e0cc76d5 +Author: Valentin Lorentz +Date: Fri Jan 11 12:04:32 2019 +0100 + + Make all worker containers depend on the scheduler runner. + +commit 4c5eea37d25e31710b6c73d5095f3cbee0800876 +Author: Valentin Lorentz +Date: Thu Jan 10 16:56:13 2019 +0100 + + Add swh-indexer-journal-client. + +commit 8deedeb75ae87e23b6758a592f435d8fc9c49243 +Author: David Douard +Date: Thu Jan 10 13:20:19 2019 +0100 + + No need to expose port 5008 in swh-scheduler-worker's Dockerfile + +commit a5a9f670b04a52a7779600a890c39f95213d46e3 +Author: David Douard +Date: Thu Jan 10 13:19:16 2019 +0100 + + Add missing swh-lister-bitbucket service + +commit 119f4f2d5529a328480d16bb6783c0bb2911ab36 +Author: David Douard +Date: Wed Jan 9 17:59:44 2019 +0100 + + Use 'swh-scheduler api-server' command to start the sheduler's API server + +commit 91288bbf2de3b52a1593f658a95fda42fa13cdd5 +Author: David Douard +Date: Wed Jan 9 16:28:10 2019 +0100 + + Fix loader config file name + + We should really consider being more consistent here: + + - loaders: ~/.config/swh/loader/.yml + - listers: ~/.config/swh/lister-.yml + +commit 06f417d3eb8894a54d0e1854237c8302377a298d +Author: David Douard +Date: Wed Jan 9 16:26:23 2019 +0100 + + task_queues configuration is not needed any more + +commit af2d06948b61d02e0a4c30b75b499c7c042607c6 +Author: David Douard +Date: Tue Dec 18 15:07:36 2018 +0100 + + Add loader workers services + +commit 72c144941f3ec0c61fd08b05f87838cec830331a +Author: David Douard +Date: Wed Jan 9 16:30:35 2019 +0100 + + Fix the example code to add a gitlab lister task + +commit fe98a7154e656b0216eaedcfdda36ecea236dbc2 +Author: David Douard +Date: Wed Jan 9 12:23:01 2019 +0100 + + Add a flower service + +commit 75c45c7aa2889e4bcac6f793d9e3361faa628050 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Jan 9 15:38:25 2019 +0100 + + swh-web: Fix missing pushd instruction + +commit 8a42fea1859c3a0893171c7e98dea28a7c07c9fd +Author: Antoine R. Dumont (@ardumont) +Date: Wed Jan 9 15:15:15 2019 +0100 + + swh-deposit: Add server's service + + Related T1459 + Related T1443 + +commit ab3cca5e7698197d330c2731a9f5f439eeec428a +Author: Valentin Lorentz +Date: Wed Jan 9 14:33:35 2019 +0100 + + Add metadata indexers. + +commit 081d478f44b62947c15b7c32aa9700f177d812a8 +Author: David Douard +Date: Tue Dec 18 14:32:46 2018 +0100 + + Print the list of python installed packages on container startup + +commit bd1b927fcefdf7e3eb663c62980c0376fac7696a +Author: Valentin Lorentz +Date: Wed Dec 26 18:18:39 2018 +0100 + + Fix build of swh-docs. + +commit 8490f7d5b25876874602d4192cc119b622e52a4d +Author: Stefano Zacchiroli +Date: Fri Dec 21 14:15:52 2018 +0100 + + README: drop mention of DB initialization + + Summary: it's no longer needed + + Reviewers: #reviewers, ardumont + + Reviewed By: #reviewers, ardumont + + Differential Revision: https://forge.softwareheritage.org/D861 + +commit 1bed89bb99dbfb41adfe140a8d6cd5621421f285 +Author: Valentin Lorentz +Date: Thu Dec 20 18:36:55 2018 +0100 + + Make the mimetype indexer work. + +commit 7917e2fde5c7a35e1552bfd9387a2197f53263a5 (origin/add-journal) +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 14:59:40 2018 +0100 + + README: Update documentation with some more precision on services + +commit 5701acebb8ac23df8d6d841b4cdaa8cc9d8ea529 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 14:59:33 2018 +0100 + + README: Reference the journal support + +commit b2f3c73311efb6b050158cad56a0cff51ca0f070 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 14:28:38 2018 +0100 + + storage-listener: Fix .pgpass rights + + Related D864 + +commit 47cc9067f1edef54c04dbf66827e1c6c91f24c53 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 14:04:19 2018 +0100 + + swh-journal-client: Fix logging and configuration + + Related T1443 + +commit 187730ea9d73a64ed5d780d618e7fdcdd7a62fcd +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 14:04:04 2018 +0100 + + publisher: Fix default configuration, directory does not exist yet + + Related T1443 + +commit e07dda8d4bbb87978b4c9601552ab2bb0d8ef027 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 13:22:54 2018 +0100 + + docker-compose: Remove unused volume + + Related T1443 + +commit 5bc984e146f712b286383cc1940780da0a9f9179 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 13:22:44 2018 +0100 + + Ignore docker-compose.override.yml + +commit ccb3294e4bbed8dd1b78f70dd6e40bcde405590a +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 13:22:10 2018 +0100 + + publisher: Fix default configuration + + Related T1443 + +commit 7dc5690ebfda476b3dadb187e42f5898520c73a7 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 00:42:25 2018 +0100 + + swh-journal-publisher: Simplify default configuration + + Related T1443 + +commit 9e2c20ed0024d0030816ca3cf1eac9c38c259a47 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 00:34:10 2018 +0100 + + swh-journal-publisher: Run in debug mode + + Related T1443 + +commit 6e51e31e1016a4703859993c19425b5cfa3e4851 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 20 00:33:42 2018 +0100 + + swh-journal-publisher: Update default configuration + + Related T1443 + +commit f8d5ab2940240006353eb19558738040a025ee93 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 23:49:08 2018 +0100 + + kafka.env: Simplify default configuration + + Related T1443 + +commit c9cba911600d0d823cce93c29a936e7a471ac8bb +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 23:48:18 2018 +0100 + + journal/listener: Fix default configuration + + Apparently using _ and . in prefix names is not recommended + +commit 24a84e493fe55688a39480832c983f96629b8793 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 20:40:51 2018 +0100 + + storage-listener: Add --verbose flag + +commit 4416afad1fa20357ce39849268d1a6a7d7f59a9c +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 15:28:10 2018 +0100 + + storage-listener: Add the missing runtime dependency + +commit 64f92fa0f21e26bace63882d7dc62293d1eb287f +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 15:24:35 2018 +0100 + + kafka: Update the default expected swh topics + +commit 81cf4e4c625f123967d6e39ec4386256f2036e6f +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 15:24:15 2018 +0100 + + swh-journal-client: Add a journal sample client + +commit 43202f74d3e6404a1f98f4606ff08b0a65b5bdf0 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 14:13:38 2018 +0100 + + Add swh-journal's publisher service + +commit ef0f4cbfdf78a4261c7e428af84c9ba59e200a04 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 14:12:33 2018 +0100 + + swh-storage-listener: Simplify default configuration + +commit 15b3403b903958ce9779d841a508c0ca9622e6ec +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 11:19:48 2018 +0100 + + swh-storage: Fix mix tab/spaces by removing all tabs + +commit c90ea18154a126b70c276e7271ac6be624cf743a +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 11:19:04 2018 +0100 + + docker-compose: Add swh-storage-listener + +commit 666c2225a6353764745a4397aaee4a2d9a634d62 +Author: Antoine R. Dumont (@ardumont) +Date: Tue Dec 18 18:00:05 2018 +0100 + + docker-compose: Make kafka start as a single node + +commit 95b98a181e01b8675d4ce8957a367ee19a7c2092 +Author: Antoine R. Dumont (@ardumont) +Date: Tue Dec 18 14:51:22 2018 +0100 + + docker-compose: Use kafka.env file + +commit e38a2967b8945829d2f1327d1aa2eec0b49cb1fd +Author: Antoine R. Dumont (@ardumont) +Date: Tue Dec 18 11:38:55 2018 +0100 + + docker-compose: Add kafka stack + +commit 357084c46082c42b034021806e369b20281538f7 +Author: David Douard +Date: Thu Dec 20 14:18:45 2018 +0100 + + Fix entrypoint of most services + + Setting .pgpass perms to 0400 prevent a container from being restarted + without being recreated (docker-compose restart fails), since the + echo xxx > .pgpass fails because the .pgpass already exists and is + read only. + +commit 61bcc9b2564392ed59e48ddb274187dde6e550a4 +Author: David Douard +Date: Thu Dec 20 13:24:36 2018 +0100 + + Fix swh-scheduler-runner config + + Ensure the scheduler.yml is properly loaded. For some reason, it seems to + require the presence of the file in both directories + /etc/softwareheritage/ AND + /etc/softwareheritage/worker/ + +commit 301a8d979e2df7485c02871dabd2ecc2f5f200ef +Author: David Douard +Date: Tue Dec 18 18:19:49 2018 +0100 + + Add a section in the README on how to create tasks to start importing content asynchronously + + also add missing elements in the list of available services. + +commit 0cff6c5c92ad6e0c7644456c60597d1b978d4551 +Author: David Douard +Date: Tue Dec 18 18:18:30 2018 +0100 + + swh-scheduler-api: add a 'swh-scheduler' command to the entrypoint + + so one can run things like: + + docker-compose run swh-scheduler-api swh-scheduler task --list + +commit f89e256d97c8ffda98daa8e3baa875bc5e08d0e9 (origin/fix-indexer) +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 19 11:20:11 2018 +0100 + + indexer-worker: Add missing dockerfile for the indexer worker + +commit 98733a8b96d18d70a74a4c11992f642b4bd5bbf5 +Author: David Douard +Date: Tue Dec 18 11:38:35 2018 +0100 + + Use 8080 as host port for the swh-web service + + since it is the main public web access. + +commit 1a7f18fe977e711afed43deb4e1e6ed24d43b530 +Author: Valentin Lorentz +Date: Tue Dec 18 14:52:08 2018 +0100 + + Add an indexer worker. + +commit eca17bc5201db0f7a229af7233aa0ecd6519d130 +Author: Valentin Lorentz +Date: Mon Dec 17 15:06:43 2018 +0100 + + Fix psycopg2 warning flooding the output. + + There's one message every 10 seconds... + +commit 58d97a091b521ef25383571ae61f985d38b62eed +Author: David Douard +Date: Tue Dec 18 10:47:23 2018 +0100 + + Duplicate the build: item in swh-scheduler-runner instead of using depends_on + +commit 3d6b7324e91c85206c709905434ef7a6f3efc9c1 +Author: David Douard +Date: Mon Dec 17 15:11:27 2018 +0100 + + Add lister workers services + + This includes the npm worker which is not yet published. + + The celery workers run under a dedicated swh user otherwise celery + complains quite loudly. + + This has a pretty annoying drawback: we cannot pip install swh packages + from volume mounted source in 'develop mode' because vcversioner will + then attempt to write 2 files in the source directory (version.txt and + the _version.py module) in which we do not have write access when + the entrypoint is run from the docker as the swh user. + + There is no easy solution for this problem, so for now, we install the + package from sources, bit NOT in develop mode (non editable). + + Related to T1443. + +commit dd130fe2321c14f6d928493ffe3eab5db00f1a9f +Author: Antoine R. Dumont (@ardumont) +Date: Mon Dec 17 16:03:33 2018 +0100 + + Fix formatting, use spaces instead of tabs + +commit dfc32e695a96af2b1dcd4dd90d1528ab53287518 +Author: Antoine R. Dumont (@ardumont) +Date: Mon Dec 17 15:51:40 2018 +0100 + + *scheduler*/entrypoint: Use unambiguous swh-scheduler service name + +commit 8b23bd35e78d98bb141de85020626466cc52c938 +Author: Antoine R. Dumont (@ardumont) +Date: Mon Dec 17 15:36:48 2018 +0100 + + swh-scheduler-api: Fix default configuration to use the right db + +commit 96b10eafbbb9e7a9942b61e7d49ac251c62bcd35 +Author: Antoine R. Dumont (@ardumont) +Date: Mon Dec 17 15:36:30 2018 +0100 + + swh-scheduler-worker: Fix default configuration to use the right db + +commit a377e8701fbaab80c21f48433e31f2ddbe893ad3 +Author: David Douard +Date: Fri Dec 14 14:26:05 2018 +0100 + + Add scheduler services + + - scheduler-api + - scheduler-listener + - scheduler-runner + + this comes with a dedicated database and a rabbitmq service. + +commit ee06ef594bb65e2343d9b638488e914f5ea4afec +Author: Valentin Lorentz +Date: Fri Dec 14 15:03:26 2018 +0100 + + Fix race condition between swh-storage and swh-storage-db. + +commit a4fe146d5893a774e1e06af9df9ab5d2a0d8f182 +Author: Valentin Lorentz +Date: Fri Dec 14 16:10:21 2018 +0100 + + Add indexer storage service. + +commit cf2492e9e1cc42ab02b56e95300da8b858a7a50d +Author: Antoine R. Dumont (@ardumont) +Date: Fri Dec 14 12:09:21 2018 +0100 + + storage/entrypoint.sh: Make the storage initialization transparent + + Related T1444 + +commit 4ab1c7da2e8eced4d57595bdb5e7412e98742a2e (origin/add-swh-web) +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 23:17:59 2018 +0100 + + swh-web/entrypoint: Add back comments + +commit b247c11ee016fbbb26d395dcdc1749ed8a7f503f +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 23:17:48 2018 +0100 + + swh-web/Dockerfile: Fix formatting + +commit 2d6c5c3d9f3d5f49e685da0f8ece1199a24201aa +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 23:17:14 2018 +0100 + + README: Mention the new webapp service + +commit 97557cb11569f7534d6f0dd6612a4241f2fc4c77 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 23:16:52 2018 +0100 + + swh-web: Remove unneeded dev instruction + +commit bf6ed65cdde6078815fc7ff25e3a6abd2c826b03 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 19:45:36 2018 +0100 + + swh-storage/entrypoint.sh: Remove tabs and use spaces + +commit cb44100fa872cf39c9b7f7dfbf5913c5d0903b0d +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 19:45:07 2018 +0100 + + swh-storage: Make the storage work again + +commit 56d6f44e2a6df108bf63c6b6ae8eb51312d0ed78 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 19:07:41 2018 +0100 + + swh-web: Make the services run against storage and objstorage + +commit 639d8647acff77baf2aae477e0b0162c87572e82 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 18:54:36 2018 +0100 + + swh-web: Make the webapp actually serve results + +commit d7ace96a3f1f661a3d84085a136a6fec1dea7575 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 18:54:24 2018 +0100 + + Remove unused dependency + +commit b7673d0a4f43873a177f59aece3a88b5793399a3 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 17:54:16 2018 +0100 + + swh-web: Bootstrap swh-web service + +commit cfaba7393aad0c822da0b109887686bae24ba098 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 16:11:11 2018 +0100 + + swh-storage/entrypoint.sh: Fix issue on initialisation step + +commit c6c34ba02a52b77488aafec0f535b6acc47b1bf9 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 15:52:27 2018 +0100 + + swh-*storage/Dockerfile: Fix missing prefix instruction RUN + +commit e6cd16ec4f576409ea3e7c6be7f9dd7de64cf0c0 +Author: David Douard +Date: Thu Dec 13 14:18:59 2018 +0100 + + refactor a bit + + - move every docker build material in a per-container dedicated directory + - use entrypoint.sh files as entrypoints + - simplify a lot the Dockerfile: we only pip install what's necessary, the + possibility of using swh packages from sources is done by detecting and + installing every repo found in /src/swh-*. It's the responsibility of the + user to mount local source directories via docker volumes. + - rewrite the docker-compose file accordingly, using env vars when needed. + +commit 51599c0ef816ec042e624698aa1305005ee33ad2 +Author: Antoine R. Dumont (@ardumont) +Date: Thu Dec 13 12:35:58 2018 +0100 + + Makefile: Add back the run dependency to build + + Related D805 + +commit cd2adc045cbfae685ac1369c8fd646c378721ce4 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 12 19:51:13 2018 +0100 + + Makefile: Revert to origin targets but explicit them + +commit 8858ab03f758b5e1c573d5a77c915752f365dae1 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 12 18:34:57 2018 +0100 + + Makefile: Untangle the commands + +commit 0a4b872dca84c83e3433ae15183d7f2520d254fe +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 12 18:33:15 2018 +0100 + + README: Finish the sentence + +commit fb00e2f3402870bbbfdbeec1cfc2528d1fb021fc +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 12 17:43:21 2018 +0100 + + Makefile: Clarify targets + +commit db9ba76235270f9b85d811f70c2cb9ebbe1b0adf +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 12 16:06:55 2018 +0100 + + Dockerfile: Use more idiomatic apt-get instructions + +commit a5b54ce0492e283391e83513d1b01a9aa8d99188 +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 12 15:38:01 2018 +0100 + + README: Reference what people can do + +commit e172f1bae913a0b64fa4a046d1d56bc8be5f165d +Author: Antoine R. Dumont (@ardumont) +Date: Wed Dec 12 15:34:04 2018 +0100 + + docker-compose: Add the service dependencies + +commit fb03c3ee5a2f3b0711cbe3a580a7c8530a6c90a4 +Merge: 405f5c1 e35ad06 +Author: Valentin Lorentz +Date: Tue Dec 11 15:04:35 2018 +0100 + + New initial structure of Docker images. + + Summary: + Creates each Docker image with the 'entrypoint' fetched locally, + and all its dependencies via PyPI. + + Reviewers: #reviewers, ardumont + + Reviewed By: #reviewers, ardumont + + Subscribers: ardumont + + Differential Revision: https://forge.softwareheritage.org/D800 + +commit e35ad06dc46f978fc870d84449fa664fb7a4a13e +Author: Valentin Lorentz +Date: Tue Dec 11 15:04:35 2018 +0100 + + New initial structure of Docker images. + + Creates each Docker image with the 'entrypoint' fetched locally, + and all its dependencies via PyPI. + +commit 405f5c18081a720a9ed3aafc9a0a6ef30120f185 +Author: Valentin Lorentz +Date: Tue Dec 11 15:34:59 2018 +0100 + + Fix repo name. + +commit aeddc46ec503df9764b8bdd5193dca0027497170 +Author: Valentin Lorentz +Date: Mon Dec 10 16:10:44 2018 +0100 + + fix typo + +commit 389705973b6da36c4e37c31eeb92f1afa8724fb8 +Author: Valentin Lorentz +Date: Fri Dec 7 17:45:04 2018 +0100 + + initial commit.