diff --git a/conf/prometheus-jmx-exporter-cassandra.yml b/conf/prometheus-jmx-exporter-cassandra.yml new file mode 100644 index 0000000..7c256d9 --- /dev/null +++ b/conf/prometheus-jmx-exporter-cassandra.yml @@ -0,0 +1,42 @@ +# see: +# - http://cassandra.apache.org/doc/latest/operating/metrics.html +# - https://blog.pythian.com/step-step-monitoring-cassandra-prometheus-grafana/ + +startDelaySeconds: 0 +hostPort: cassandra-seed:7199 +username: +password: +#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi +ssl: false +lowercaseOutputName: false +lowercaseOutputLabelNames: false +whitelistObjectNames: ["org.apache.cassandra.metrics:*"] +blacklistObjectNames: [] +rules: +- pattern: org.apache.cassandra.metrics<>(Count|Value) + name: cassandra_$1_$3 + labels: + address: "$2" +- pattern: org.apache.cassandra.metrics<>(Mean) + name: cassandra_$1_$2_$3 +- pattern: org.apache.cassandra.net<>(DownEndpointCount) + name: cassandra_$1_$2 +- pattern: org.apache.cassandra.metrics<>(Count|Mean|95thPercentile) + name: cassandra_$1_$3_$4 + labels: + "$1": "$2" +- pattern: org.apache.cassandra.metrics<>(Count|Mean|95thPercentile) + name: cassandra_$1_$4_$5 + labels: + "keyspace": "$2" + "table": "$3" +- pattern: org.apache.cassandra.metrics<>(Count|Mean|95thPercentile) + name: cassandra_$1_$3_$4 + labels: + "type": "$2" +- pattern: org.apache.cassandra.metrics<>(Count|Value) + name: cassandra_$1_$5 + labels: + "$1": "$4" + "$2": "$3" diff --git a/conf/prometheus-jmx-exporter-logging.properties b/conf/prometheus-jmx-exporter-logging.properties new file mode 100644 index 0000000..0f4c31f --- /dev/null +++ b/conf/prometheus-jmx-exporter-logging.properties @@ -0,0 +1,6 @@ +handlers=java.util.logging.ConsoleHandler +# uncomment this to get logs: +#java.util.logging.ConsoleHandler.level=ALL +#io.prometheus.jmx.level=ALL +#io.prometheus.jmx.shaded.io.prometheus.jmx.level=ALL + diff --git a/conf/prometheus.yml b/conf/prometheus.yml index 9e10744..f342c98 100644 --- a/conf/prometheus.yml +++ b/conf/prometheus.yml @@ -1,17 +1,22 @@ # my global config global: scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. # scrape_timeout is set to the global default (10s). scrape_configs: - job_name: prometheus static_configs: - targets: - localhost:9090 metrics_path: /prometheus/metrics - job_name: statsd-exporter static_configs: - targets: - prometheus-statsd-exporter:9102 + + - job_name: jmx-exporter-cassandra + static_configs: + - targets: + - prometheus-jmx-exporter-cassandra:5556 diff --git a/docker-compose.cassandra.yml b/docker-compose.cassandra.yml index 6724e54..d9bcbb4 100644 --- a/docker-compose.cassandra.yml +++ b/docker-compose.cassandra.yml @@ -1,46 +1,62 @@ version: '2' services: cassandra-seed: # This container starts a Cassandra instance that must be used as the # contact-point for clients. This container will then make the client # discover other cassandra containers. # This container must not be scaled up; scale up th 'cassandra' # container instead. image: cassandra env_file: - ./env/cassandra.env entrypoint: /swh_entrypoint.sh volumes: - "./services/cassandra/swh_entrypoint.sh:/swh_entrypoint.sh:ro" - "./conf/cassandra.yaml:/cassandra.yaml:ro" cassandra: # Additional Cassandra instance(s), which may be scaled up, but not # down. They will automatically connect to 'cassandra-seed', and # 'cassandra-seed' will tell clients to connect to these 'cassandra' # containers to load-balance. image: cassandra entrypoint: /swh_entrypoint.sh volumes: - "./services/cassandra/swh_entrypoint.sh:/swh_entrypoint.sh:ro" - "./conf/cassandra.yaml:/cassandra.yaml:ro" env_file: - ./env/cassandra.env + prometheus: + # just to add the dep on the cassandra-jmx-exporter-cassandra + depends_on: + - prometheus-statsd-exporter + - prometheus-jmx-exporter-cassandra + + prometheus-jmx-exporter-cassandra: + image: sscaling/jmx-prometheus-exporter + environment: + JVM_OPTS: "-Djava.util.logging.config.file=/logging.properties" + volumes: + - "./conf/prometheus-jmx-exporter-cassandra.yml:/opt/jmx_exporter/config.yml:ro" + - "./conf/prometheus-jmx-exporter-logging.properties:/logging.properties:ro" + ports: + - "5556:5556" + swh-storage: volumes: # note: you need to be on the cassandra-backend2 branch - "/home/dev/swh-environment/swh-storage:/src/swh-storage" - "/home/dev/swh-environment/swh-model:/src/swh-model" - "./conf/storage_cassandra.yml:/storage.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" depends_on: - swh-storage-db - cassandra-seed - swh-objstorage - kafka environment: CASSANDRA_SEED: cassandra-seed STORAGE_BACKEND: cassandra PYTHONUNBUFFERED: 1 diff --git a/docker-compose.yml b/docker-compose.yml index 8875249..4b81a65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,382 +1,386 @@ version: '2' services: amqp: image: rabbitmq:3.6-management ports: - 5072:5672 # flower: # image: mher/flower # command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower # ports: # - 5055:5555 # depends_on: # - amqp zookeeper: image: wurstmeister/zookeeper restart: always kafka: image: wurstmeister/kafka ports: - "5092:9092" env_file: ./env/kafka.env depends_on: - zookeeper kafka-manager: image: hlebalbau/kafka-manager:stable ports: - "5093:9000" environment: ZK_HOSTS: zookeeper:2181 APPLICATION_SECRET: random-secret command: -Dpidfile.path=/dev/null prometheus: image: prom/prometheus + depends_on: + - prometheus-statsd-exporter command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro" restart: unless-stopped prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" restart: unless-stopped grafana: image: grafana/grafana restart: unless-stopped + depends_on: + - prometheus environment: GF_SERVER_ROOT_URL: http://localhost:5080/grafana volumes: - "./conf/grafana/provisioning:/etc/grafana/provisioning:ro" - "./conf/grafana/dashboards:/var/lib/grafana/dashboards" nginx: image: nginx volumes: - "./conf/nginx.conf:/etc/nginx/nginx.conf:ro" ports: - 5080:5080 # Scheduler swh-scheduler-db: image: postgres:11 env_file: - ./env/scheduler-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-scheduler-api: image: swh/stack build: ./ env_file: - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh depends_on: - swh-scheduler-db ports: - 5008:5008 volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler-api/entrypoint.sh:/entrypoint.sh:ro" swh-scheduler-listener: image: swh/stack build: ./ env_file: - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh command: listener depends_on: - swh-scheduler-api - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler-worker/entrypoint.sh:/entrypoint.sh:ro" swh-scheduler-runner: image: swh/stack build: ./ env_file: - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh command: runner -p 10 depends_on: - swh-scheduler-api - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler-worker/entrypoint.sh:/entrypoint.sh:ro" # Graph storage swh-storage-db: image: postgres:11 env_file: - ./env/storage-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-storage: image: swh/stack build: ./ ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage - kafka env_file: - ./env/storage-db.env environment: SWH_CONFIG_FILENAME: /storage.yml STORAGE_BACKEND: postgresql entrypoint: /entrypoint.sh volumes: - "./conf/storage.yml:/storage.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" # Object storage swh-objstorage: build: ./ image: swh/stack ports: - 5003:5003 environment: SWH_CONFIG_FILENAME: /objstorage.yml entrypoint: /entrypoint.sh volumes: - "./conf/objstorage.yml:/objstorage.yml:ro" - "./services/swh-objstorage/entrypoint.sh:/entrypoint.sh:ro" # Indexer storage swh-idx-storage-db: image: postgres:11 env_file: - ./env/indexers-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-idx-storage: image: swh/stack build: ./ ports: - 5007:5007 depends_on: - swh-idx-storage-db env_file: - ./env/indexers-db.env environment: SWH_CONFIG_FILENAME: /indexer_storage.yml entrypoint: /entrypoint.sh volumes: - "./conf/indexer_storage.yml:/indexer_storage.yml:ro" - "./services/swh-indexer-storage/entrypoint.sh:/entrypoint.sh:ro" # Web interface swh-web: build: ./ image: swh/stack ports: - 5004:5004 depends_on: - swh-objstorage - swh-storage - swh-idx-storage environment: VERBOSITY: 3 DJANGO_SETTINGS_MODULE: swh.web.settings.development SWH_CONFIG_FILENAME: /web.yml entrypoint: /entrypoint.sh volumes: - "./conf/web.yml:/web.yml:ro" - "./services/swh-web/entrypoint.sh:/entrypoint.sh:ro" swh-deposit-db: image: postgres:11 env_file: - ./env/deposit-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-deposit: image: swh/stack build: ./ ports: - 5006:5006 depends_on: - swh-deposit-db - swh-scheduler-api env_file: - ./env/deposit-db.env - ./env/deposit.env entrypoint: /entrypoint.sh volumes: - "./conf/deposit.yml:/deposit.yml:ro" - "./services/swh-deposit/entrypoint.sh:/entrypoint.sh:ro" swh-vault-db: image: postgres:11 env_file: - ./env/vault-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-vault-api: image: swh/stack build: ./ env_file: - ./env/vault-db.env environment: SWH_CONFIG_FILENAME: /vault-api.yml command: server ports: - 5005:5005 depends_on: - swh-vault-db - swh-objstorage - swh-storage - swh-scheduler-api entrypoint: /entrypoint.sh volumes: - "./conf/vault-api.yml:/vault-api.yml:ro" - "./services/swh-vault/entrypoint.sh:/entrypoint.sh:ro" swh-vault-worker: image: swh/stack build: ./ command: worker environment: SWH_CONFIG_FILENAME: /cooker.yml depends_on: - swh-vault-api - swh-storage entrypoint: /entrypoint.sh volumes: - "./conf/vault-worker.yml:/cooker.yml:ro" - "./services/swh-vault/entrypoint.sh:/entrypoint.sh:ro" # Lister Celery workers swh-listers-db: image: postgres:11 env_file: - ./env/listers-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-lister: image: swh/stack build: ./ env_file: - ./env/listers-db.env - ./env/listers.env user: swh environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: listers SWH_CONFIG_FILENAME: /lister.yml depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp entrypoint: /entrypoint.sh volumes: - "./conf/lister.yml:/lister.yml:ro" - "./services/swh-listers-worker/entrypoint.sh:/entrypoint.sh:ro" # Loader Celery workers swh-loader: image: swh/stack build: ./ env_file: - ./env/listers.env user: swh environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: loader SWH_CONFIG_FILENAME: /loader.yml entrypoint: /entrypoint.sh depends_on: - swh-storage - amqp volumes: - "./conf/loader.yml:/loader.yml:ro" - "./services/swh-loaders-worker/entrypoint.sh:/entrypoint.sh:ro" # Indexer Celery workers swh-indexer: image: swh/stack build: ./ user: swh env_file: - ./env/indexers-db.env - ./env/indexers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 entrypoint: /entrypoint.sh depends_on: - swh-scheduler-runner - swh-idx-storage - swh-storage - swh-objstorage - amqp volumes: - "./conf/indexer.yml:/indexer.yml:ro" - "./services/swh-indexer-worker/entrypoint.sh:/entrypoint.sh:ro" # Journal related swh-indexer-journal-client: image: swh/stack build: ./ entrypoint: /entrypoint.sh depends_on: - kafka - swh-storage - swh-scheduler-api volumes: - "./conf/indexer_journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" - "./services/swh-indexer-journal-client/entrypoint.sh:/entrypoint.sh:ro"