diff --git a/docs/infrastructure/service-urls.rst b/docs/infrastructure/service-urls.rst index a320a0a..d925ae8 100644 --- a/docs/infrastructure/service-urls.rst +++ b/docs/infrastructure/service-urls.rst @@ -1,191 +1,191 @@ Service urls ##################### This section regroups the urls of the services .. toctree:: :maxdepth: 2 :titlesonly: Staging ------- Try to use the staging environment as far as possible for your tests Public urls ~~~~~~~~~~~ +---------------------------------------+-------------------------------------------+ | Service | URL | +=======================================+===========================================+ | swh-web | https://webapp.staging.swh.network | +---------------------------------------+-------------------------------------------+ | swh-deposit | https://deposit.staging.swh.network | +---------------------------------------+-------------------------------------------+ | swh-objstorage read-only (for mirror) | https://objstorage.staging.swh.network | +---------------------------------------+-------------------------------------------+ -| Journal TLS | broker0.journal.staging.swh.network:9093 | +| Journal TLS | broker1.journal.staging.swh.network:9093 | +---------------------------------------+-------------------------------------------+ Internal services ~~~~~~~~~~~~~~~~~ +--------------------------+------------------------------------------------------+--------+------------+ | Service | URL | VPN[1] | Private[2] | +==========================+======================================================+========+============+ | swh-storage | http://storage1.internal.staging.swh.network:5002 | | X | +--------------------------+------------------------------------------------------+--------+------------+ | swh-storage read-only | http://webapp.internal.staging.swh.network:5002 | X | | +--------------------------+------------------------------------------------------+--------+------------+ | swh-objstorage | http://storage1.internal.staging.swh.network:5003 | | X | +--------------------------+------------------------------------------------------+--------+------------+ | swh-objstorage read-only | http://objstorage0.internal.staging.swh.network:5003 | X | | +--------------------------+------------------------------------------------------+--------+------------+ | swh-scheduler | http://scheduler0.internal.staging.swh.network:5008 | X | | +--------------------------+------------------------------------------------------+--------+------------+ | swh-counters | http://counters0.internal.staging.swh.network:5011 | X | | +--------------------------+------------------------------------------------------+--------+------------+ | swh-search | http://webapp.internal.staging.swh.network:5010 | X | | +--------------------------+------------------------------------------------------+--------+------------+ | swh-search | http://search0.internal.staging.swh.network:5010 | | X | +--------------------------+------------------------------------------------------+--------+------------+ | swh-vault | http://vault.internal.staging.swh.network:5005 | | X | +--------------------------+------------------------------------------------------+--------+------------+ -| Journal plaintext | journal0.internal.staging.swh.network:9092 | | X | +| Journal plaintext | journal1.internal.staging.swh.network:9092 | | X | +--------------------------+------------------------------------------------------+--------+------------+ -| Journal internal TLS | journal0.internal.staging.swh.network:9094 | | X | +| Journal internal TLS | journal1.internal.staging.swh.network:9094 | | X | +--------------------------+------------------------------------------------------+--------+------------+ SWH backends ~~~~~~~~~~~~ +--------------------+---------------------------------------------------------+--------+------------+ | Backend | URL | VPN[1] | Private[2] | +====================+=========================================================+========+============+ | RabbitMq GUI | http://scheduler0.internal.staging.swh.network:15672 | X | | +--------------------+---------------------------------------------------------+--------+------------+ | archive database | db1.internal.staging.swh.network:5432/swh | X | | +--------------------+---------------------------------------------------------+--------+------------+ | webapp database | db1.internal.staging.swh.network:5432/swh-web | X | | +--------------------+---------------------------------------------------------+--------+------------+ | deposit database | db1.internal.staging.swh.network:5432/swh-deposit | X | | +--------------------+---------------------------------------------------------+--------+------------+ | vault database | db1.internal.staging.swh.network:5432/swh-vault | X | | +--------------------+---------------------------------------------------------+--------+------------+ | scheduler database | db1.internal.staging.swh.network:5432/swh-scheduler | X | | +--------------------+---------------------------------------------------------+--------+------------+ | lister database | db1.internal.staging.swh.network:5432/swh-lister | X | | +--------------------+---------------------------------------------------------+--------+------------+ | swh-search ES | http://search-esnode0.internal.staging.swh.network:9200 | | X | +--------------------+---------------------------------------------------------+--------+------------+ | Counters redis | counters0.internal.staging.swh.network:6379 | | X | +--------------------+---------------------------------------------------------+--------+------------+ Production ---------- .. _public-urls-1: Public urls ~~~~~~~~~~~ +---------------------------------------+-----------------------------------------------+ | Service | URL | +=======================================+===============================================+ | swh-web | https://archive.softwareheritage.org | +---------------------------------------+-----------------------------------------------+ | swh-deposit | https://deposit.softwareheritage.org | +---------------------------------------+-----------------------------------------------+ | swh-objstorage read-only (for mirror) | N/A | +---------------------------------------+-----------------------------------------------+ | Journal TLS | broker[1-4].journal.softwareheritage.org:9093 | +---------------------------------------+-----------------------------------------------+ .. _internal-services-1: Internal services ~~~~~~~~~~~~~~~~~ +--------------------------+----------------------------------------------------------------+--------+------------+ | Service | URL | VPN[1] | Private[2] | +==========================+================================================================+========+============+ | swh-web test/validation | https://webapp1.internal.softwareheritage.org | X | | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-storage | http://saam.internal.softwareheritage.org:5002 | | X | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-storage read-only | http://webapp1.internal.softwareheritage.org:5002 | X | | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-storage read-only | http://moma.internal.softwareheritage.org:5002 | X | | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-objstorage | http://saam.internal.softwareheritage.org:5003 | | X | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-objstorage read-only | N/A | | | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-scheduler | http://saatchi.internal.softwareheritage.org:5008 | X | | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-counters | http://counters1.internal.softwareheritage.org:5011 | X | | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-search | http://webapp1.internal.softwareheritage.org:5010 | X | | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-search | http://moma.internal.softwareheritage.org:5010 | X | | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-search | http://search1.internal.softwareheritage.org:5010 | | X | +--------------------------+----------------------------------------------------------------+--------+------------+ | swh-vault | http://vangogh.euwest.azure.internal.softwareheritage.org:5005 | | X | +--------------------------+----------------------------------------------------------------+--------+------------+ | Journal plaintext | kafka[1-4].internal.softwareheritage.org:9092 | | X | +--------------------------+----------------------------------------------------------------+--------+------------+ | Journal internal TLS | kafka[1-4].internal.softwareheritage.org:9094 | X | | +--------------------------+----------------------------------------------------------------+--------+------------+ .. _swh-backends-1: SWH backends ~~~~~~~~~~~~ +--------------------------+-----------------------------------------------------------------------+--------+------------+ | Backend | URL | VPN[1] | Private[2] | +==========================+=======================================================================+========+============+ | RabbitMq GUI | http://saatchi.internal.softwareheritage.org:15672 | X | | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | archive database replica | somerset.internal.softwareheritage.org:5432/softwareheritage | X | | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | archive database main | belvedere.internal.softwareheritage.org:5432/softwareheritage | X | | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | webapp database main | belvedere.internal.softwareheritage.org:5432/swh-web | X | | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | scheduler database | belvedere.internal.softwareheritage.org:5432/swh-scheduler | X | | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | lister database | belvedere.internal.softwareheritage.org:5432/swh-lister | X | | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | deposit database | belvedere.internal.softwareheritage.org:5432/softwareheritage-deposit | X | | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | vault database | belvedere.internal.softwareheritage.org:5432/swh-vault | X | | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | swh-search ES | http://search-esnode[1-3].internal.softwareheritage.org:9200 | | X | +--------------------------+-----------------------------------------------------------------------+--------+------------+ | Counters redis | counters1.internal.softwareheritage.org:6379 | | X | +--------------------------+-----------------------------------------------------------------------+--------+------------+ Other tools ----------- +-------------------+-------------------------------------------------------+--------------------+--------+------------+ | Tool | URL | Public | VPN[1] | Private[2] | +===================+=======================================================+====================+========+============+ | grafana | https://grafana.softwareheritage.org | X | | | +-------------------+-------------------------------------------------------+--------------------+--------+------------+ | Kibana | http://kibana0.internal.softwareheritage.org:5601 | | X | | +-------------------+-------------------------------------------------------+--------------------+--------+------------+ | Log Elasticsearch | http://search[1-3].internal.softwareheritage.org:9200 | | X | | +-------------------+-------------------------------------------------------+--------------------+--------+------------+ | C.M.A.K. | http://getty.internal.softwareheritage.org:9000 | | X | | +-------------------+-------------------------------------------------------+--------------------+--------+------------+ | Sentry | https://sentry.softwareheritage.org | X (authentication) | | | +-------------------+-------------------------------------------------------+--------------------+--------+------------+ [1] VPN: URL only accessible when connected to the SoftwareHeritage VPN [2] Private: URL only accessible from the internal network, i.e nor public neither accessible through the VPN. diff --git a/sysadm/images/staging-environment.uml b/sysadm/images/staging-environment.uml index b506893..3f466e8 100644 --- a/sysadm/images/staging-environment.uml +++ b/sysadm/images/staging-environment.uml @@ -1,231 +1,229 @@ ' Related to T2920 @startuml title Staging environment actor user actor mirror cloud VLAN1300 { interface "archive.staging.swh.network" as publicarchive interface "deposit.staging.swh.network" as publicdeposit - interface "broker0.journal.staging.swh.network" as publicjournal + interface "broker1.journal.staging.swh.network" as publicjournal } node firewall { } node pompidou { node "rp0.internal.staging.swh.network" as rp0 { component varnish { boundary 80 } component hitch { boundary 443 } } node "vault.internal.staging.swh.network" { component nginx as nginxvault { boundary 5005 as vault } component "swh-vault" as vaultgunicorn file "/srv/softwareheritage/vault_cache" as vaultcache vaultgunicorn -right-> vault vaultgunicorn --> vaultcache } together { - node "journal0.internal.staging.swh.network" as journal0 { - component kafka { - boundary 9094 - boundary 9093 - boundary 9092 as kafkainternal - } - } node "webapp.internal.staging.swh.network" as webapp0 { component apache as apacheservicewebapp { boundary 9080 as apachewebapp } component webapp apachewebapp -left- webapp } node "deposit.internal.staging.swh.network" as deposit { component apache as apacheservicedeposit { boundary 9080 as apachedeposit } component depositgunicorn apachedeposit -right-> depositgunicorn } node "objstorage0.internal.staging.swh.network" as objstorage0 { component nginx as nginxobjstoragero { boundary 5003 as objstoragero } component "objstorage\nread-only" as objstoragerogunicorn objstoragero --> objstoragerogunicorn } depositgunicorn -down-> apachewebapp - webapp0 -left-> journal0 objstorage0 -[hidden]right->deposit deposit -[hidden]right->webapp0 - journal0 -[hidden]right->webapp0 - journal0 -[hidden]right->deposit } together { component workers { component "swh-loaders" as swhloader component "swh-listers" as swhlister component "swh-deposit" as swhdeposit component "swh-indexer" as swhindexer component "swh-vault-cooker" as swhvaultcooker } note top of workers: worker0.internal.staging.swh.network\nworker1.internal.staging.swh.network\nworker2.internal.staging.swh.network node "scheduler0.internal.staging.swh.network" as scheduler { component nginx as schedulernginx { boundary 5008 as swhscheduler } component "swh-scheduler" as swhschedulergunicorn component "swh-scheduler-runner" as swhschedulerrunner component "swh-scheduler-listener" as swhschedulerlistener component "swh-scheduler-journal-client" as swhschedulerjournalclient queue rabbitmq swhschedulergunicorn --> swhscheduler swhschedulerrunner --> rabbitmq } workers -[hidden]--> scheduler } together { node "search0.internal.staging.swh.network" as search { component nginx as nginxsearch { boundary 5010 as swhsearch } component "swh-search" as swhsearchinternal component "swh-search-journal-objects" as searchjournalobjects component "swh-search-journal-indexed" as searchjournalindexed swhsearch -down-> swhsearchinternal searchjournalobjects -[hidden]right-> swhsearchinternal } node "search-esnode0.internal.staging.swh.network" as esnode { component elasticsearch { boundary 9200 as essearch } } } } note top of pompidou: Proxmox server dedicated to staging user --> publicarchive user --> publicdeposit mirror --> publicjournal publicarchive --> firewall publicdeposit --> firewall publicjournal --> firewall firewall -down-> 80 firewall -down-> 443 443 -left- 80 -firewall -down-> 9094 ' note top: Authenticated\nconnection together { node "storage1.internal.staging.swh.network" as storage1 { folder "/srv/softwareheritage/objects" as objects component nginx { boundary 5003 as objstorage boundary 5002 as storage boundary 5007 as indexerstorage } + component kafka { + boundary 9094 + boundary 9093 + boundary 9092 as kafkainternal + } + + component "swh-objstorage" as objstoragegunicorn component "swh-storage" as storagegunicorn component "swh-indexer-storage" as indexerstoragegunicorn objstoragegunicorn --> objects objstorage --> objstoragegunicorn storage --> storagegunicorn indexerstorage --> indexerstoragegunicorn } node "db1internal.staging.swh.network" as db1 { component postgresql { database swh as swhdb database "swh-scheduler" as schedulerdb database "swh-deposit" as depositdb database "swh-indexer" as indexerdb database "swh-vault" as vaultdb database "swh-lister" as listerdb } } } +firewall -down-> 9094 + vaultgunicorn --> vaultdb vaultgunicorn --> swhscheduler vaultgunicorn --> storage swhschedulergunicorn --> schedulerdb varnish -down-> apachewebapp webapp -down-> storage webapp -down-> objstorage webapp -down-> swhsearch swhschedulerjournalclient --> kafkainternal swhschedulerjournalclient --> schedulerdb swhschedulerrunner --> schedulerdb swhschedulerlistener --> schedulerdb swhloader -down-> storage swhloader -up-> rabbitmq swhlister -up-> listerdb swhlister -up-> schedulerdb swhlister -up-> rabbitmq swhindexer -up-> indexerdb swhindexer -up-> rabbitmq swhvaultcooker -down-> storage swhvaultcooker --> vault swhvaultcooker -up-> rabbitmq swhdeposit --> storage objstoragerogunicorn --> objstorage varnish -down-> objstoragero storagegunicorn -up-> swhdb indexerstoragegunicorn -up-> indexerdb indexerstoragegunicorn -down-> kafkainternal storagegunicorn -down-> kafkainternal searchjournalobjects --> essearch searchjournalindexed -down-> essearch swhsearchinternal -down-> essearch searchjournalobjects -up-> kafkainternal searchjournalindexed -up-> kafkainternal search -[hidden]left-> objstorage0 depositgunicorn --> depositdb depositgunicorn --> storage depositgunicorn --> swhscheduler varnish --> apachedeposit db1 -up->pompidou storage1 -up-> db1 @enduml