diff --git a/proxmox/terraform/production/cassandra.tf b/proxmox/terraform/production/cassandra.tf --- a/proxmox/terraform/production/cassandra.tf +++ b/proxmox/terraform/production/cassandra.tf @@ -17,6 +17,13 @@ network { plugin = "canal" } + services { + kubelet { + extra_binds = [ + "/srv/prometheus:/srv/prometheus" # prometheus datastore on mngmt nodes + ] + } + } } } @@ -155,3 +162,22 @@ output "rancher_node_cassandra3_summary" { value = module.rancher_node_cassandra3.summary } + +# Install the cassandra operator +# https://github.com/k8ssandra/cass-operator +resource "rancher2_catalog_v2" "k8ssandra" { + cluster_id = rancher2_cluster.production_cassandra.id + name = "k8ssandra" + url = "https://helm.k8ssandra.io/stable" +} + +resource "rancher2_app_v2" "cass_operator" { + cluster_id = rancher2_cluster.production_cassandra.id + name = "cass-operator" + namespace = "cass-operator" + repo_name = "k8ssandra" + chart_name = "cass-operator" + chart_version = "0.37.0" + values = "replicaCount: 2" +} + diff --git a/proxmox/terraform/production/terraform.tfstate b/proxmox/terraform/production/terraform.tfstate --- a/proxmox/terraform/production/terraform.tfstate +++ b/proxmox/terraform/production/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.2.4", - "serial": 203, + "serial": 214, "lineage": "e2912de8-415d-4c35-8ee5-757008306227", "outputs": { "production_cassandra_cluster_command": { @@ -36,6 +36,86 @@ } }, "resources": [ + { + "mode": "managed", + "type": "rancher2_app_v2", + "name": "cass_operator", + "provider": "provider[\"registry.terraform.io/rancher/rancher2\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "annotations": { + "objectset.rio.cattle.io/applied": "H4sIAAAAAAAA/8xXbVPbuhL+K7m6X+5MbSfOCwTP3JlygZZeeoAB+jJt+bCWNrYaWdJIsoHD5L+fkUwg5kBaaHumfHJWq319nl1xTSp0wMABya4JSKkcOK6k9T9V/hWps+gSw1VCwTmBCVd9zkhGShRVDFqT6FE9dSHRxEUzJxnpN2nUO+CS/fcUqUH3zWsSKiQZsWXiPSUGBYLFpEkTCtbGSqMBp0zSDL/LlNVAvb3OZbKICDUYMj7jFVoHlSaZrIWIiIAcxdo6lGBLkhGcMhhT2NpK6WyK48lsC0dTtjEb5ZtDNpuMp3SI040t9N5usupG0YofjjAiIYcTnKFBSdGS7PM1Ac3fo7FcSZKRJiURyYWi8yOvuosCXTiZgbAYEaqkM0oINCRzpsaIzLn0LbztxJNqXYf2TzbYhOLmIKYsHcdj2BrGMJgN4+lgvDnOR2meb2ySxfkiIlYj9VWkJRjnP56BONBacPR+98f2zfbNX396+fHDsE9d+fbFwaipmv9f7J9+rPv723uv5l+12zs4Pdw9c4fF6acX1c77/T1j360By+8F6/THYN1FiO8baH0nSZM0TQYkIgwtNVy3gCEHdY5GokPbWxrrXZSclr0SJBNoe67Enjaq4d4Ql0UPJOtVIKHACiWJSKlCgqVz2mb9fsFdWecJVVV/PrUWJDPQvw/xCrh0wCWaFt5YARf3s3rpMWMdXHpjd6DdAWt7R8tgzxD8WW3E02JYRHdeb3Vihg0Kr2NfFkoVAgujam27ARws1b/fOVmcPzoKrKpNS/On1XCt8t2XM4h9X+9+4KO9Z+c8Iu5K+7gC5WigJolIc4ucQTLaTAYeYg2IGgNtgVXc+vMPmJdKzYMQJeTCczaMIT9pVVUp+XY5VhcRmdVC+DIcNWgMZ94tiUghVA4ijAxRW4fmlCq9aohXUKA/17UQx0pwekUy8mZ2qNyxQYvS09Bgwa0z/oQpOkeTcLUivnPZTnuDWlnuVLjweJXv1O4bcFCEWdwSaxGtwKhNPCR0+4WXUGmB730NPQrCQdzOyE4JAulXivHGTykaayoZI9FSetiCSRvFaup7FvvIW7BFxF5Zh9UuzqAW7uSuMGH/PVDjdTe+p22/W8wt2VbDfUAUEa3YdmcfLYLsFGltuLvaUdLhpWvlBgM9dlQtHclSL7hlrj+3f7tEDAI7kuLqRCn3igts413uZFPLbfvazxeSbUwmo/GN6FBJf6Gj9s76XR60givTcIrblLbB3N+qvmVNdQEGj0+P97qsXERh490uhlFEuJwpb6S7Gd7pwgDDHlUetg5JRGbcWLeLWqirsJqHg+EwHmzG6fAsHWaTSTZJPxH/jlqntZUNp58CsYAFOPy71+HcF/lF/uvzTXxZb5CMJsng/D/LacerIrElR8GsX4w5sMKHZh242nrqLx0/7+210tQHX123D6lOB57hyu+fR83vKDnjxR+gH7Ect/vXxDQoPs2RyYEmULtSGf5nwEwyn4ZidkJoGXeiBD4WBP0Zxv/HJeOy+Pk+1kUuEBg+sT8/7PVXuftGBX9truudP80raG07DtohUuEzGXa++mL5B5fVurPuFhkuVh5a/sdyjPl/jPyUR/Yapc8naAwiYuuqAm/terFY/BUAAP//SHMdHswPAAA", + "objectset.rio.cattle.io/id": "helm-app", + "objectset.rio.cattle.io/owner-gvk": "/v1, Kind=Secret", + "objectset.rio.cattle.io/owner-name": "sh.helm.release.v1.cass-operator.v2", + "objectset.rio.cattle.io/owner-namespace": "cass-operator" + }, + "chart_name": "cass-operator", + "chart_version": "0.37.0", + "cleanup_on_fail": false, + "cluster_id": "c-pcndd", + "cluster_name": "production-cassandra", + "disable_hooks": false, + "disable_open_api_validation": false, + "force_upgrade": false, + "id": "c-pcndd.cass-operator/cass-operator", + "labels": { + "objectset.rio.cattle.io/hash": "e8da4ca991cf8e45f9e38d6f3b72df548c2e869e" + }, + "name": "cass-operator", + "namespace": "cass-operator", + "project_id": null, + "repo_name": "k8ssandra", + "system_default_registry": "", + "timeouts": null, + "values": "global:\n cattle:\n clusterId: c-pcndd\n clusterName: production-cassandra\n systemDefaultRegistry: \"\"\n systemDefaultRegistry: \"\"\nreplicaCount: 2\n", + "wait": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "rancher2_cluster.production_cassandra" + ] + } + ] + }, + { + "mode": "managed", + "type": "rancher2_catalog_v2", + "name": "k8ssandra", + "provider": "provider[\"registry.terraform.io/rancher/rancher2\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "annotations": {}, + "ca_bundle": "", + "cluster_id": "c-pcndd", + "enabled": true, + "git_branch": "", + "git_repo": "", + "id": "c-pcndd.k8ssandra", + "insecure": false, + "labels": {}, + "name": "k8ssandra", + "resource_version": "229815", + "secret_name": null, + "secret_namespace": null, + "service_account": "", + "service_account_namespace": "", + "timeouts": null, + "url": "https://helm.k8ssandra.io/stable" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "rancher2_cluster.production_cassandra" + ] + } + ] + }, { "mode": "managed", "type": "rancher2_cluster", @@ -49,9 +129,14 @@ "aks_config": [], "aks_config_v2": [], "annotations": { - "authz.management.cattle.io/creator-role-bindings": "{\"created\":[\"cluster-owner\"],\"required\":[\"cluster-owner\"]}" + "authz.management.cattle.io/creator-role-bindings": "{\"created\":[\"cluster-owner\"],\"required\":[\"cluster-owner\"]}", + "lifecycle.cattle.io/create.cluster-agent-controller-cleanup": "true", + "lifecycle.cattle.io/create.cluster-provisioner-controller": "true", + "lifecycle.cattle.io/create.cluster-scoped-gc": "true", + "lifecycle.cattle.io/create.mgmt-cluster-rbac-remove": "true", + "provisioner.cattle.io/ke-driver-update": "updated" }, - "ca_cert": null, + "ca_cert": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM0VENDQWNtZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFTTVJBd0RnWURWUVFERXdkcmRXSmwKTFdOaE1CNFhEVEl5TURjeE1URTRNalUwTWxvWERUTXlNRGN3T0RFNE1qVTBNbG93RWpFUU1BNEdBMVVFQXhNSAphM1ZpWlMxallUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU1EcUkxR2ZVdkdFCkRnUGUycTFkRU1RRUxLWGl5dWsvWTRNWm1MZFl4b3M5aEdXN2R4NExEb3phLzFCUlZGY1cwNXZXYVU0Q3dQaUMKeGF1ZkYzbXhuc2ErZnhJQ1kzUlB0QzVmak83cmtIekdEZ2Q2dWc1VDFLSzZ4YVRBMjhndmxUN21pek9CK3djSgo0ajhMNUZ6enZmWFpLUlNZRlJ6UnI5Q3J5NnNGR0pkSG42c295aUx2TW1kTlh5UWNwSmV1Tm1nMmdSU0dGNGh3CmZ3M1Z4RzdFT3N6MFh3bkpDWGxCNE1zdzd5YzV6TFErN3hJMjhIYkFsNGFxcUxKRUwvTUlxMjM0cWNTU3Z5NE8KZ1gxc2FiZllHV0dHZnVwZUxjc3BqSnVOMkt5bWxpbDJsUXo0QU5YbkZsbDlNamVHVVVzZVFiRWR0NkZ0cTUwVgpyejJoNDhTWG1aTUNBd0VBQWFOQ01FQXdEZ1lEVlIwUEFRSC9CQVFEQWdLa01BOEdBMVVkRXdFQi93UUZNQU1CCkFmOHdIUVlEVlIwT0JCWUVGR2M3RmFJSCt5WENpUHZ6RmgvMmNkQWpuQVVzTUEwR0NTcUdTSWIzRFFFQkN3VUEKQTRJQkFRQjdEcW54cVVDb3ZtQ0JCYW1pSUJGL3YwQWtydWl5NlFXbE1uUWNJL055TytZdmFWcy9SUUZ2bEFiMwo1MnhxOEx5cFN0RHBIcVRoVjNCaXhaS3RaQ2hUODdBcnNqL3N5U0Y5dU82YytJSFoxTXlJWXdOaU1PbjR0Vk82CkExWmtVZTdRamZvOTF2WElFN2Q2TXZJRDMzL3BFcDlxNis3WU1TKys3NWNlN0RRYTBadURtd2Nna1FxeFQ5UDIKZU1nSmJOYytYU3ZOWEREOWFWTVJzOEF5TEROZ2w4ZXZQRjJuOEJVT1BxV2RWUFc1Y2swYzNkYnVHeHdCNkpGZQovTVA5NmJiVGtjWU9DRWZ5WjVrWXBEc244OFhZK1c5NWVBYWMrVUF1M3RGa3V3Uk9oTUJvUmF2K09LQldXc1hECmpVUnNyekNEdEVVSjBLd3NCZTI3SThXMUJwa0EKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", "cluster_auth_endpoint": [ { "ca_certs": "", @@ -79,9 +164,9 @@ "windows_node_command": "PowerShell -NoLogo -NonInteractive -Command \"\u0026 {docker run -v c:\\:c:\\host rancher/rancher-agent:v2.6.4 bootstrap --server https://rancher.euwest.azure.internal.softwareheritage.org --token 4s5g6zqh5rdl8v9gzstr55sb5wxsh62hrz6gfgmh5sgnk5p8259n98 --ca-checksum 8850dd89eb7f29a70c0d50a2b389bf1950808a9dff4062c66ab806b80b988bac | iex}\"" } ], - "cluster_template_answers": null, + "cluster_template_answers": [], "cluster_template_id": null, - "cluster_template_questions": null, + "cluster_template_questions": [], "cluster_template_revision_id": null, "default_pod_security_policy_template_id": null, "default_project_id": "c-pcndd:p-qx9vf", @@ -89,14 +174,14 @@ "desired_agent_image": null, "desired_auth_image": null, "docker_root_dir": "/var/lib/docker", - "driver": "", + "driver": "rancherKubernetesEngine", "eks_config": [], - "eks_config_v2": null, + "eks_config_v2": [], "enable_cluster_alerting": false, "enable_cluster_istio": null, "enable_cluster_monitoring": false, "enable_network_policy": false, - "fleet_workspace_name": null, + "fleet_workspace_name": "fleet-default", "gke_config": [], "gke_config_v2": [], "id": "c-pcndd", @@ -104,7 +189,8 @@ "k3s_config": [], "kube_config": "apiVersion: v1\nkind: Config\nclusters:\n- name: \"production-cassandra\"\n cluster:\n server: \"https://rancher.euwest.azure.internal.softwareheritage.org/k8s/clusters/c-pcndd\"\n certificate-authority-data: \"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJwekNDQ\\\n VUyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQTdNUnd3R2dZRFZRUUtFeE5rZVc1aGJXbGoKY\\\n kdsemRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR1Z1WlhJdFkyRXdIa\\\n GNOTWpJdwpOREV5TWpJek56RTBXaGNOTXpJd05EQTVNakl6TnpFMFdqQTdNUnd3R2dZRFZRUUtFe\\\n E5rZVc1aGJXbGpiR2x6CmRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR\\\n 1Z1WlhJdFkyRXdXVEFUQmdjcWhrak8KUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVMvTnRpTWV4NDY0K\\\n 0R3RytuQ2svQXdJM2VDSjNGOHdQNitUZytNTjFnaQo3OWJWeEIwd1RTYjZmL3ZXK3NRdEIzdEUrS\\\n k5iaU1mU2xuQUtaVlBjQjA0T28wSXdRREFPQmdOVkhROEJBZjhFCkJBTUNBcVF3RHdZRFZSMFRBU\\\n UgvQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVeVUyT1lacnBkbCtWYkIzSkpUcHUKMktiVnF0NHdDZ\\\n 1lJS29aSXpqMEVBd0lEU0FBd1JRSWdRY25JcnJCSzdUbFBBdlRKb3EwNmp5QVplcURzTWI1LwpKZ\\\n Th4QVFnTnV5d0NJUURaWDA2NDJ4NXh2N1lFdmZ4eWJ0SnlRWVY3aTlJZ1llNzlKVG9SaVBQTkZBP\\\n T0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\"\n\nusers:\n- name: \"production-cassandra\"\n user:\n token: \"kubeconfig-u-rzh79fcvz9:h9spl2p692sjlsgsst58tw9l2k42v7z4s7zdxhpvlvcmz584gf8jjh\"\n\n\ncontexts:\n- name: \"production-cassandra\"\n context:\n user: \"production-cassandra\"\n cluster: \"production-cassandra\"\n\ncurrent-context: \"production-cassandra\"\n", "labels": { - "cattle.io/creator": "norman" + "cattle.io/creator": "norman", + "provider.cattle.io": "rke" }, "name": "production-cassandra", "oke_config": [], @@ -113,7 +199,7 @@ { "addon_job_timeout": 45, "addons": "", - "addons_include": null, + "addons_include": [], "authentication": [ { "sans": [], @@ -138,12 +224,12 @@ ], "cloud_provider": [ { - "aws_cloud_provider": null, - "azure_cloud_provider": null, - "custom_cloud_provider": null, - "name": null, - "openstack_cloud_provider": null, - "vsphere_cloud_provider": null + "aws_cloud_provider": [], + "azure_cloud_provider": [], + "custom_cloud_provider": "", + "name": "", + "openstack_cloud_provider": [], + "vsphere_cloud_provider": [] } ], "dns": [], @@ -164,7 +250,7 @@ "update_strategy": [] } ], - "kubernetes_version": "v1.21.13-rancher1-1", + "kubernetes_version": "v1.22.10-rancher1-1", "monitoring": [ { "node_selector": {}, @@ -254,7 +340,9 @@ "cluster_dns_server": "", "cluster_domain": "", "extra_args": {}, - "extra_binds": [], + "extra_binds": [ + "/srv/prometheus:/srv/prometheus" + ], "extra_env": [], "fail_swap_on": false, "generate_serving_certificate": false, @@ -302,7 +390,30 @@ "win_prefix_path": "" } ], - "scheduled_cluster_scan": null, + "scheduled_cluster_scan": [ + { + "enabled": false, + "scan_config": [ + { + "cis_scan_config": [ + { + "debug_master": false, + "debug_worker": false, + "override_benchmark_version": "rke-cis-1.5", + "override_skip": [], + "profile": "permissive" + } + ] + } + ], + "schedule_config": [ + { + "cron_schedule": "0 0 * * *", + "retention": 24 + } + ] + } + ], "system_project_id": "c-pcndd:p-nvptk", "timeouts": null, "windows_prefered_cluster": false