diff --git a/proxmox/terraform/staging/rancher.tf b/proxmox/terraform/staging/rancher.tf new file mode 100644 --- /dev/null +++ b/proxmox/terraform/staging/rancher.tf @@ -0,0 +1,43 @@ +# This declares terraform manifests to provision vms and register containers within +# those to a rancher (clusters management service) instance. + +# Each software has the following responsibilities: +# - proxmox: provision vms (with docker dependency) +# - rancher: installs kube cluster within containers (running on vms) + +# Requires (RANCHER_ACCESS_KEY and RANCHER_SECRET_KEY) in your shell environment +# $ cat ~/.config/terraform/swh/setup.sh +# ... +# key_entry=operations/rancher/azure/elastic-loader-lister-keys +# export RANCHER_ACCESS_KEY=$(swhpass ls $key_entry | head -1 | cut -d: -f1) +# export RANCHER_SECRET_KEY=$(swhpass ls $key_entry | head -1 | cut -d: -f2) +provider "rancher2" { + api_url = "https://rancher.euwest.azure.internal.softwareheritage.org/v3" + # for now + insecure = true +} + +# Plan: +# - create cluster with terraform +# - Create nodes as usual through terraform +# - Retrieve the registration command (out of the cluster creation step) to provide new +# node + +resource "rancher2_cluster" "staging-workers" { + name = "staging-workers" + description = "staging workers cluster" + rke_config { + network { + plugin = "canal" + } + } +} + +output "rancher2_cluster_summary" { + sensitive = true + value = rancher2_cluster.staging-workers.kube_config +} + +output "rancher2_cluster_command" { + value = rancher2_cluster.staging-workers.cluster_registration_token[0].node_command +} diff --git a/proxmox/terraform/staging/terraform.tfstate b/proxmox/terraform/staging/terraform.tfstate --- a/proxmox/terraform/staging/terraform.tfstate +++ b/proxmox/terraform/staging/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.1.8", - "serial": 203, + "serial": 206, "lineage": "e2912de8-415d-4c35-8ee5-757008306227", "outputs": { "counters0_summary": { @@ -24,6 +24,15 @@ "value": "\nhostname: objstorage0\nfqdn: objstorage0.internal.staging.swh.network\nnetwork: ip=192.168.130.110/24,gw=192.168.130.1 macaddrs=5E:28:EA:7D:50:0D\nvmid: 102\n", "type": "string" }, + "rancher2_cluster_command": { + "value": "sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent:v2.6.4 --server https://rancher.euwest.azure.internal.softwareheritage.org --token zbtc5cl2l6qfp6jrwmw6jqztmtq6kbdv5m8jk2w7qktq8gb9knb6zq --ca-checksum 8850dd89eb7f29a70c0d50a2b389bf1950808a9dff4062c66ab806b80b988bac", + "type": "string" + }, + "rancher2_cluster_summary": { + "value": "apiVersion: v1\nkind: Config\nclusters:\n- name: \"staging-workers\"\n cluster:\n server: \"https://rancher.euwest.azure.internal.softwareheritage.org/k8s/clusters/c-t85mz\"\n certificate-authority-data: \"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJwekNDQ\\\n VUyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQTdNUnd3R2dZRFZRUUtFeE5rZVc1aGJXbGoKY\\\n kdsemRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR1Z1WlhJdFkyRXdIa\\\n GNOTWpJdwpOREV5TWpJek56RTBXaGNOTXpJd05EQTVNakl6TnpFMFdqQTdNUnd3R2dZRFZRUUtFe\\\n E5rZVc1aGJXbGpiR2x6CmRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR\\\n 1Z1WlhJdFkyRXdXVEFUQmdjcWhrak8KUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVMvTnRpTWV4NDY0K\\\n 0R3RytuQ2svQXdJM2VDSjNGOHdQNitUZytNTjFnaQo3OWJWeEIwd1RTYjZmL3ZXK3NRdEIzdEUrS\\\n k5iaU1mU2xuQUtaVlBjQjA0T28wSXdRREFPQmdOVkhROEJBZjhFCkJBTUNBcVF3RHdZRFZSMFRBU\\\n UgvQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVeVUyT1lacnBkbCtWYkIzSkpUcHUKMktiVnF0NHdDZ\\\n 1lJS29aSXpqMEVBd0lEU0FBd1JRSWdRY25JcnJCSzdUbFBBdlRKb3EwNmp5QVplcURzTWI1LwpKZ\\\n Th4QVFnTnV5d0NJUURaWDA2NDJ4NXh2N1lFdmZ4eWJ0SnlRWVY3aTlJZ1llNzlKVG9SaVBQTkZBP\\\n T0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\"\n\nusers:\n- name: \"staging-workers\"\n user:\n token: \"kubeconfig-user-92j6d5nng8:ld4njkk6wh8gwnnczjvcnlrprnf9k227c2w4d4cnh8j67phlxbhjlv\"\n\n\ncontexts:\n- name: \"staging-workers\"\n context:\n user: \"staging-workers\"\n cluster: \"staging-workers\"\n\ncurrent-context: \"staging-workers\"\n", + "type": "string", + "sensitive": true + }, "rp0_summary": { "value": "\nhostname: rp0\nfqdn: rp0.internal.staging.swh.network\nnetwork: ip=192.168.130.20/24,gw=192.168.130.1 macaddrs=4A:80:47:5D:DF:73\nvmid: 129\n", "type": "string" @@ -70,6 +79,282 @@ } }, "resources": [ + { + "mode": "managed", + "type": "rancher2_cluster", + "name": "staging-workers", + "provider": "provider[\"registry.terraform.io/rancher/rancher2\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "agent_env_vars": [], + "aks_config": [], + "aks_config_v2": [], + "annotations": { + "authz.management.cattle.io/creator-role-bindings": "{\"created\":[\"cluster-owner\"],\"required\":[\"cluster-owner\"]}" + }, + "ca_cert": null, + "cluster_auth_endpoint": [ + { + "ca_certs": "", + "enabled": false, + "fqdn": "" + } + ], + "cluster_monitoring_input": [], + "cluster_registration_token": [ + { + "annotations": {}, + "cluster_id": "c-t85mz", + "command": "kubectl apply -f https://rancher.euwest.azure.internal.softwareheritage.org/v3/import/zbtc5cl2l6qfp6jrwmw6jqztmtq6kbdv5m8jk2w7qktq8gb9knb6zq_c-t85mz.yaml", + "id": "c-t85mz:default-token", + "insecure_command": "curl --insecure -sfL https://rancher.euwest.azure.internal.softwareheritage.org/v3/import/zbtc5cl2l6qfp6jrwmw6jqztmtq6kbdv5m8jk2w7qktq8gb9knb6zq_c-t85mz.yaml | kubectl apply -f -", + "insecure_node_command": "", + "insecure_windows_node_command": "", + "labels": { + "cattle.io/creator": "norman" + }, + "manifest_url": "https://rancher.euwest.azure.internal.softwareheritage.org/v3/import/zbtc5cl2l6qfp6jrwmw6jqztmtq6kbdv5m8jk2w7qktq8gb9knb6zq_c-t85mz.yaml", + "name": "default-token", + "node_command": "sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent:v2.6.4 --server https://rancher.euwest.azure.internal.softwareheritage.org --token zbtc5cl2l6qfp6jrwmw6jqztmtq6kbdv5m8jk2w7qktq8gb9knb6zq --ca-checksum 8850dd89eb7f29a70c0d50a2b389bf1950808a9dff4062c66ab806b80b988bac", + "token": "zbtc5cl2l6qfp6jrwmw6jqztmtq6kbdv5m8jk2w7qktq8gb9knb6zq", + "windows_node_command": "PowerShell -NoLogo -NonInteractive -Command \"\u0026 {docker run -v c:\\:c:\\host rancher/rancher-agent:v2.6.4 bootstrap --server https://rancher.euwest.azure.internal.softwareheritage.org --token zbtc5cl2l6qfp6jrwmw6jqztmtq6kbdv5m8jk2w7qktq8gb9knb6zq --ca-checksum 8850dd89eb7f29a70c0d50a2b389bf1950808a9dff4062c66ab806b80b988bac | iex}\"" + } + ], + "cluster_template_answers": null, + "cluster_template_id": null, + "cluster_template_questions": null, + "cluster_template_revision_id": null, + "default_pod_security_policy_template_id": null, + "default_project_id": "c-t85mz:p-x7ls4", + "description": "staging workers cluster", + "desired_agent_image": null, + "desired_auth_image": null, + "docker_root_dir": "/var/lib/docker", + "driver": "", + "eks_config": [], + "eks_config_v2": null, + "enable_cluster_alerting": false, + "enable_cluster_istio": null, + "enable_cluster_monitoring": false, + "enable_network_policy": false, + "fleet_workspace_name": "fleet-default", + "gke_config": [], + "gke_config_v2": [], + "id": "c-t85mz", + "istio_enabled": false, + "k3s_config": [], + "kube_config": "apiVersion: v1\nkind: Config\nclusters:\n- name: \"staging-workers\"\n cluster:\n server: \"https://rancher.euwest.azure.internal.softwareheritage.org/k8s/clusters/c-t85mz\"\n certificate-authority-data: \"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJwekNDQ\\\n VUyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQTdNUnd3R2dZRFZRUUtFeE5rZVc1aGJXbGoKY\\\n kdsemRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR1Z1WlhJdFkyRXdIa\\\n GNOTWpJdwpOREV5TWpJek56RTBXaGNOTXpJd05EQTVNakl6TnpFMFdqQTdNUnd3R2dZRFZRUUtFe\\\n E5rZVc1aGJXbGpiR2x6CmRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR\\\n 1Z1WlhJdFkyRXdXVEFUQmdjcWhrak8KUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVMvTnRpTWV4NDY0K\\\n 0R3RytuQ2svQXdJM2VDSjNGOHdQNitUZytNTjFnaQo3OWJWeEIwd1RTYjZmL3ZXK3NRdEIzdEUrS\\\n k5iaU1mU2xuQUtaVlBjQjA0T28wSXdRREFPQmdOVkhROEJBZjhFCkJBTUNBcVF3RHdZRFZSMFRBU\\\n UgvQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVeVUyT1lacnBkbCtWYkIzSkpUcHUKMktiVnF0NHdDZ\\\n 1lJS29aSXpqMEVBd0lEU0FBd1JRSWdRY25JcnJCSzdUbFBBdlRKb3EwNmp5QVplcURzTWI1LwpKZ\\\n Th4QVFnTnV5d0NJUURaWDA2NDJ4NXh2N1lFdmZ4eWJ0SnlRWVY3aTlJZ1llNzlKVG9SaVBQTkZBP\\\n T0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\"\n\nusers:\n- name: \"staging-workers\"\n user:\n token: \"kubeconfig-user-92j6d5nng8:ld4njkk6wh8gwnnczjvcnlrprnf9k227c2w4d4cnh8j67phlxbhjlv\"\n\n\ncontexts:\n- name: \"staging-workers\"\n context:\n user: \"staging-workers\"\n cluster: \"staging-workers\"\n\ncurrent-context: \"staging-workers\"\n", + "labels": { + "cattle.io/creator": "norman" + }, + "name": "staging-workers", + "oke_config": [], + "rke2_config": [], + "rke_config": [ + { + "addon_job_timeout": 45, + "addons": "", + "addons_include": null, + "authentication": [ + { + "sans": [], + "strategy": "x509" + } + ], + "authorization": [ + { + "mode": null, + "options": null + } + ], + "bastion_host": [ + { + "address": "", + "port": "", + "ssh_agent_auth": false, + "ssh_key": "", + "ssh_key_path": "", + "user": "" + } + ], + "cloud_provider": [ + { + "aws_cloud_provider": null, + "azure_cloud_provider": null, + "custom_cloud_provider": null, + "name": null, + "openstack_cloud_provider": null, + "vsphere_cloud_provider": null + } + ], + "dns": [], + "enable_cri_dockerd": false, + "ignore_docker_version": true, + "ingress": [ + { + "default_backend": true, + "dns_policy": "", + "extra_args": {}, + "http_port": 0, + "https_port": 0, + "network_mode": "", + "node_selector": {}, + "options": {}, + "provider": "nginx", + "tolerations": [], + "update_strategy": [] + } + ], + "kubernetes_version": "v1.22.7-rancher1-2", + "monitoring": [ + { + "node_selector": {}, + "options": {}, + "provider": "metrics-server", + "replicas": 1, + "tolerations": [], + "update_strategy": [] + } + ], + "network": [ + { + "calico_network_provider": [], + "canal_network_provider": [], + "flannel_network_provider": [], + "mtu": 0, + "options": {}, + "plugin": "canal", + "tolerations": [], + "weave_network_provider": [] + } + ], + "nodes": [], + "prefix_path": "", + "private_registries": [], + "services": [ + { + "etcd": [ + { + "backup_config": [ + { + "enabled": true, + "interval_hours": 12, + "retention": 6, + "s3_backup_config": [], + "safe_timestamp": false, + "timeout": 300 + } + ], + "ca_cert": "", + "cert": "", + "creation": "12h", + "external_urls": [], + "extra_args": { + "election-timeout": "5000", + "heartbeat-interval": "500" + }, + "extra_binds": [], + "extra_env": [], + "gid": 0, + "image": "", + "key": "", + "path": "", + "retention": "72h", + "snapshot": false, + "uid": 0 + } + ], + "kube_api": [ + { + "admission_configuration": {}, + "always_pull_images": false, + "audit_log": [], + "event_rate_limit": [], + "extra_args": {}, + "extra_binds": [], + "extra_env": [], + "image": "", + "pod_security_policy": false, + "secrets_encryption_config": [], + "service_cluster_ip_range": "", + "service_node_port_range": "30000-32767" + } + ], + "kube_controller": [ + { + "cluster_cidr": null, + "extra_args": null, + "extra_binds": null, + "extra_env": null, + "image": null, + "service_cluster_ip_range": null + } + ], + "kubelet": [ + { + "cluster_dns_server": "", + "cluster_domain": "", + "extra_args": {}, + "extra_binds": [], + "extra_env": [], + "fail_swap_on": false, + "generate_serving_certificate": false, + "image": "", + "infra_container_image": "" + } + ], + "kubeproxy": [ + { + "extra_args": null, + "extra_binds": null, + "extra_env": null, + "image": null + } + ], + "scheduler": [ + { + "extra_args": null, + "extra_binds": null, + "extra_env": null, + "image": null + } + ] + } + ], + "ssh_agent_auth": false, + "ssh_cert_path": "", + "ssh_key_path": "", + "upgrade_strategy": [ + { + "drain": false, + "drain_input": [ + { + "delete_local_data": false, + "force": false, + "grace_period": -1, + "ignore_daemon_sets": true, + "timeout": 120 + } + ], + "max_unavailable_controlplane": "1", + "max_unavailable_worker": "10%" + } + ], + "win_prefix_path": "" + } + ], + "scheduled_cluster_scan": null, + "system_project_id": "c-t85mz:p-spm4t", + "timeouts": null, + "windows_prefered_cluster": false + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9" + } + ] + }, { "module": "module.counters0", "mode": "managed", diff --git a/proxmox/terraform/versions.tf b/proxmox/terraform/versions.tf --- a/proxmox/terraform/versions.tf +++ b/proxmox/terraform/versions.tf @@ -5,6 +5,10 @@ source = "telmate/proxmox" version = "2.9.3" } + rancher2 = { + source = "rancher/rancher2" + version = "1.23.0" + } } }