diff --git a/proxmox/terraform/staging/rancher.tf b/proxmox/terraform/staging/rancher.tf new file mode 100644 --- /dev/null +++ b/proxmox/terraform/staging/rancher.tf @@ -0,0 +1,46 @@ +# This declares terraform manifests to provision vms and register containers within +# those to a rancher (clusters management service) instance. + +# Each software has the following responsibilities: +# - proxmox: provision vms (with docker dependency) +# - rancher: installs kube cluster within containers (running on vms) + +# Requires (RANCHER_ACCESS_KEY and RANCHER_SECRET_KEY) in your shell environment +# $ cat ~/.config/terraform/swh/setup.sh +# ... +# key_entry=operations/rancher/azure/elastic-loader-lister-keys +# export RANCHER_ACCESS_KEY=$(swhpass ls $key_entry | head -1 | cut -d: -f1) +# export RANCHER_SECRET_KEY=$(swhpass ls $key_entry | head -1 | cut -d: -f2) +provider "rancher2" { + api_url = "https://rancher.euwest.azure.internal.softwareheritage.org/v3" + # for now + insecure = true +} + +# Plan: +# - create cluster with terraform +# - Create nodes as usual through terraform +# - Retrieve the registration command (out of the cluster creation step) to provide new +# node as a remote-exec step + +# Create a new rancher2 RKE Cluster +resource "rancher2_cluster" "test-rke" { + name = "test-rke" + description = "Test rke" + rke_config { + network { + plugin = "canal" + } + } +} + +output "rancher2_cluster_summary" { + sensitive = true + value = rancher2_cluster.test-rke.kube_config +} + +output "rancher2_cluster_command" { + value = rancher2_cluster.test-rke.cluster_registration_token[0].node_command +} + +} diff --git a/proxmox/terraform/staging/terraform.tfstate b/proxmox/terraform/staging/terraform.tfstate --- a/proxmox/terraform/staging/terraform.tfstate +++ b/proxmox/terraform/staging/terraform.tfstate @@ -67,9 +67,295 @@ "worker3_summary": { "value": "\nhostname: worker3\nfqdn: worker3.internal.staging.swh.network\nnetwork: ip=192.168.130.103/24,gw=192.168.130.1 macaddrs=1A:F8:1A:2C:12:E1\nvmid: 137\n", "type": "string" + }, + "rancher2_cluster_command": { + "value": "sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent:v2.6.4 --server https://rancher.euwest.azure.internal.softwareheritage.org --token z284ctbtl4kqfbqhgm2k9g6b5chrpwkz29ljjt5xb2m2bqcp5p4st2 --ca-checksum 8850dd89eb7f29a70c0d50a2b389bf1950808a9dff4062c66ab806b80b988bac", + "type": "string" + }, + "rancher2_cluster_summary": { + "value": "apiVersion: v1\nkind: Config\nclusters:\n- name: \"test-rke\"\n cluster:\n server: \"https://rancher.euwest.azure.internal.softwareheritage.org/k8s/clusters/c-dqlbs\"\n certificate-authority-data: \"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJwekNDQ\\\n VUyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQTdNUnd3R2dZRFZRUUtFeE5rZVc1aGJXbGoKY\\\n kdsemRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR1Z1WlhJdFkyRXdIa\\\n GNOTWpJdwpOREV5TWpJek56RTBXaGNOTXpJd05EQTVNakl6TnpFMFdqQTdNUnd3R2dZRFZRUUtFe\\\n E5rZVc1aGJXbGpiR2x6CmRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR\\\n 1Z1WlhJdFkyRXdXVEFUQmdjcWhrak8KUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVMvTnRpTWV4NDY0K\\\n 0R3RytuQ2svQXdJM2VDSjNGOHdQNitUZytNTjFnaQo3OWJWeEIwd1RTYjZmL3ZXK3NRdEIzdEUrS\\\n k5iaU1mU2xuQUtaVlBjQjA0T28wSXdRREFPQmdOVkhROEJBZjhFCkJBTUNBcVF3RHdZRFZSMFRBU\\\n UgvQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVeVUyT1lacnBkbCtWYkIzSkpUcHUKMktiVnF0NHdDZ\\\n 1lJS29aSXpqMEVBd0lEU0FBd1JRSWdRY25JcnJCSzdUbFBBdlRKb3EwNmp5QVplcURzTWI1LwpKZ\\\n Th4QVFnTnV5d0NJUURaWDA2NDJ4NXh2N1lFdmZ4eWJ0SnlRWVY3aTlJZ1llNzlKVG9SaVBQTkZBP\\\n T0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\"\n\nusers:\n- name: \"test-rke\"\n user:\n token: \"kubeconfig-user-92j6dkbwpr:282vdcj2tw82wm7j2cfd4qq76xxvzsz9jhvbcxjjk5kvddjkf4rqw9\"\n\n\ncontexts:\n- name: \"test-rke\"\n context:\n user: \"test-rke\"\n cluster: \"test-rke\"\n\ncurrent-context: \"test-rke\"\n", + "type": "string", + "sensitive": true } }, "resources": [ + { + "mode": "managed", + "type": "rancher2_cluster", + "name": "test-rke", + "provider": "provider[\"registry.terraform.io/rancher/rancher2\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "agent_env_vars": [], + "aks_config": [], + "aks_config_v2": [], + "annotations": { + "authz.management.cattle.io/creator-role-bindings": "{\"created\":[\"cluster-owner\"],\"required\":[\"cluster-owner\"]}", + "lifecycle.cattle.io/create.cluster-agent-controller-cleanup": "true", + "lifecycle.cattle.io/create.cluster-scoped-gc": "true", + "lifecycle.cattle.io/create.mgmt-cluster-rbac-remove": "true" + }, + "ca_cert": null, + "cluster_auth_endpoint": [ + { + "ca_certs": "", + "enabled": false, + "fqdn": "" + } + ], + "cluster_monitoring_input": [], + "cluster_registration_token": [ + { + "annotations": {}, + "cluster_id": "c-dqlbs", + "command": "kubectl apply -f https://rancher.euwest.azure.internal.softwareheritage.org/v3/import/z284ctbtl4kqfbqhgm2k9g6b5chrpwkz29ljjt5xb2m2bqcp5p4st2_c-dqlbs.yaml", + "id": "c-dqlbs:default-token", + "insecure_command": "curl --insecure -sfL https://rancher.euwest.azure.internal.softwareheritage.org/v3/import/z284ctbtl4kqfbqhgm2k9g6b5chrpwkz29ljjt5xb2m2bqcp5p4st2_c-dqlbs.yaml | kubectl apply -f -", + "insecure_node_command": "", + "insecure_windows_node_command": "", + "labels": {}, + "manifest_url": "https://rancher.euwest.azure.internal.softwareheritage.org/v3/import/z284ctbtl4kqfbqhgm2k9g6b5chrpwkz29ljjt5xb2m2bqcp5p4st2_c-dqlbs.yaml", + "name": "default-token", + "node_command": "sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent:v2.6.4 --server https://rancher.euwest.azure.internal.softwareheritage.org --token z284ctbtl4kqfbqhgm2k9g6b5chrpwkz29ljjt5xb2m2bqcp5p4st2 --ca-checksum 8850dd89eb7f29a70c0d50a2b389bf1950808a9dff4062c66ab806b80b988bac", + "token": "z284ctbtl4kqfbqhgm2k9g6b5chrpwkz29ljjt5xb2m2bqcp5p4st2", + "windows_node_command": "PowerShell -NoLogo -NonInteractive -Command \"\u0026 {docker run -v c:\\:c:\\host rancher/rancher-agent:v2.6.4 bootstrap --server https://rancher.euwest.azure.internal.softwareheritage.org --token z284ctbtl4kqfbqhgm2k9g6b5chrpwkz29ljjt5xb2m2bqcp5p4st2 --ca-checksum 8850dd89eb7f29a70c0d50a2b389bf1950808a9dff4062c66ab806b80b988bac | iex}\"" + } + ], + "cluster_template_answers": [], + "cluster_template_id": null, + "cluster_template_questions": [], + "cluster_template_revision_id": null, + "default_pod_security_policy_template_id": null, + "default_project_id": "c-dqlbs:p-shtmm", + "description": "Test rke", + "desired_agent_image": null, + "desired_auth_image": null, + "docker_root_dir": "/var/lib/docker", + "driver": "rancherKubernetesEngine", + "eks_config": [], + "eks_config_v2": [], + "enable_cluster_alerting": false, + "enable_cluster_istio": null, + "enable_cluster_monitoring": false, + "enable_network_policy": false, + "fleet_workspace_name": "fleet-default", + "gke_config": [], + "gke_config_v2": [], + "id": "c-dqlbs", + "istio_enabled": false, + "k3s_config": [], + "kube_config": "apiVersion: v1\nkind: Config\nclusters:\n- name: \"test-rke\"\n cluster:\n server: \"https://rancher.euwest.azure.internal.softwareheritage.org/k8s/clusters/c-dqlbs\"\n certificate-authority-data: \"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJwekNDQ\\\n VUyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQTdNUnd3R2dZRFZRUUtFeE5rZVc1aGJXbGoKY\\\n kdsemRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR1Z1WlhJdFkyRXdIa\\\n GNOTWpJdwpOREV5TWpJek56RTBXaGNOTXpJd05EQTVNakl6TnpFMFdqQTdNUnd3R2dZRFZRUUtFe\\\n E5rZVc1aGJXbGpiR2x6CmRHVnVaWEl0YjNKbk1Sc3dHUVlEVlFRREV4SmtlVzVoYldsamJHbHpkR\\\n 1Z1WlhJdFkyRXdXVEFUQmdjcWhrak8KUFFJQkJnZ3Foa2pPUFFNQkJ3TkNBQVMvTnRpTWV4NDY0K\\\n 0R3RytuQ2svQXdJM2VDSjNGOHdQNitUZytNTjFnaQo3OWJWeEIwd1RTYjZmL3ZXK3NRdEIzdEUrS\\\n k5iaU1mU2xuQUtaVlBjQjA0T28wSXdRREFPQmdOVkhROEJBZjhFCkJBTUNBcVF3RHdZRFZSMFRBU\\\n UgvQkFVd0F3RUIvekFkQmdOVkhRNEVGZ1FVeVUyT1lacnBkbCtWYkIzSkpUcHUKMktiVnF0NHdDZ\\\n 1lJS29aSXpqMEVBd0lEU0FBd1JRSWdRY25JcnJCSzdUbFBBdlRKb3EwNmp5QVplcURzTWI1LwpKZ\\\n Th4QVFnTnV5d0NJUURaWDA2NDJ4NXh2N1lFdmZ4eWJ0SnlRWVY3aTlJZ1llNzlKVG9SaVBQTkZBP\\\n T0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\"\n\nusers:\n- name: \"test-rke\"\n user:\n token: \"kubeconfig-user-92j6dkbwpr:282vdcj2tw82wm7j2cfd4qq76xxvzsz9jhvbcxjjk5kvddjkf4rqw9\"\n\n\ncontexts:\n- name: \"test-rke\"\n context:\n user: \"test-rke\"\n cluster: \"test-rke\"\n\ncurrent-context: \"test-rke\"\n", + "labels": { + "cattle.io/creator": "norman" + }, + "name": "test-rke", + "oke_config": [], + "rke2_config": [], + "rke_config": [ + { + "addon_job_timeout": 45, + "addons": "", + "addons_include": [], + "authentication": [ + { + "sans": [], + "strategy": "x509" + } + ], + "authorization": [ + { + "mode": null, + "options": null + } + ], + "bastion_host": [ + { + "address": "", + "port": "", + "ssh_agent_auth": false, + "ssh_key": "", + "ssh_key_path": "", + "user": "" + } + ], + "cloud_provider": [ + { + "aws_cloud_provider": [], + "azure_cloud_provider": [], + "custom_cloud_provider": "", + "name": "", + "openstack_cloud_provider": [], + "vsphere_cloud_provider": [] + } + ], + "dns": [], + "enable_cri_dockerd": false, + "ignore_docker_version": true, + "ingress": [ + { + "default_backend": true, + "dns_policy": "", + "extra_args": {}, + "http_port": 0, + "https_port": 0, + "network_mode": "", + "node_selector": {}, + "options": {}, + "provider": "nginx", + "tolerations": [], + "update_strategy": [] + } + ], + "kubernetes_version": "v1.22.7-rancher1-2", + "monitoring": [ + { + "node_selector": {}, + "options": {}, + "provider": "metrics-server", + "replicas": 1, + "tolerations": [], + "update_strategy": [] + } + ], + "network": [ + { + "calico_network_provider": [], + "canal_network_provider": [], + "flannel_network_provider": [], + "mtu": 0, + "options": {}, + "plugin": "canal", + "tolerations": [], + "weave_network_provider": [] + } + ], + "nodes": [], + "prefix_path": "", + "private_registries": [], + "services": [ + { + "etcd": [ + { + "backup_config": [ + { + "enabled": true, + "interval_hours": 12, + "retention": 6, + "s3_backup_config": [], + "safe_timestamp": false, + "timeout": 300 + } + ], + "ca_cert": "", + "cert": "", + "creation": "12h", + "external_urls": [], + "extra_args": { + "election-timeout": "5000", + "heartbeat-interval": "500" + }, + "extra_binds": [], + "extra_env": [], + "gid": 0, + "image": "", + "key": "", + "path": "", + "retention": "72h", + "snapshot": false, + "uid": 0 + } + ], + "kube_api": [ + { + "admission_configuration": {}, + "always_pull_images": false, + "audit_log": [], + "event_rate_limit": [], + "extra_args": {}, + "extra_binds": [], + "extra_env": [], + "image": "", + "pod_security_policy": false, + "secrets_encryption_config": [], + "service_cluster_ip_range": "", + "service_node_port_range": "30000-32767" + } + ], + "kube_controller": [ + { + "cluster_cidr": null, + "extra_args": null, + "extra_binds": null, + "extra_env": null, + "image": null, + "service_cluster_ip_range": null + } + ], + "kubelet": [ + { + "cluster_dns_server": "", + "cluster_domain": "", + "extra_args": {}, + "extra_binds": [], + "extra_env": [], + "fail_swap_on": false, + "generate_serving_certificate": false, + "image": "", + "infra_container_image": "" + } + ], + "kubeproxy": [ + { + "extra_args": null, + "extra_binds": null, + "extra_env": null, + "image": null + } + ], + "scheduler": [ + { + "extra_args": null, + "extra_binds": null, + "extra_env": null, + "image": null + } + ] + } + ], + "ssh_agent_auth": false, + "ssh_cert_path": "", + "ssh_key_path": "", + "upgrade_strategy": [ + { + "drain": false, + "drain_input": [ + { + "delete_local_data": false, + "force": false, + "grace_period": -1, + "ignore_daemon_sets": true, + "timeout": 120 + } + ], + "max_unavailable_controlplane": "1", + "max_unavailable_worker": "10%" + } + ], + "win_prefix_path": "" + } + ], + "scheduled_cluster_scan": [], + "system_project_id": "c-dqlbs:p-tbq4f", + "timeouts": null, + "windows_prefered_cluster": false + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9" + } + ] + }, { "module": "module.counters0", "mode": "managed", diff --git a/proxmox/terraform/versions.tf b/proxmox/terraform/versions.tf --- a/proxmox/terraform/versions.tf +++ b/proxmox/terraform/versions.tf @@ -5,6 +5,10 @@ source = "telmate/proxmox" version = "2.9.3" } + rancher2 = { + source = "rancher/rancher2" + version = "1.23.0" + } } }