diff --git a/azure/terraform/cassandra-replayers.tf b/azure/terraform/cassandra-replayers.tf index 26c8cf8..9fc1119 100644 --- a/azure/terraform/cassandra-replayers.tf +++ b/azure/terraform/cassandra-replayers.tf @@ -1,141 +1,140 @@ variable "cassandra_replay_servers" { default = 4 } resource "azurerm_resource_group" "euwest-cassandra-replay" { name = "euwest-cassandra-replay" location = "westeurope" tags = { environment = "Cassandra" } } locals { cassandra_replay_servers = { for i in range(var.cassandra_replay_servers): format("cassandra-replay%02d", i + 1) => { datadisks = {} } } } resource "azurerm_network_interface" "cassandra-replayer-interface" { for_each = local.cassandra_replay_servers name = format("%s-interface", each.key) location = "westeurope" resource_group_name = azurerm_resource_group.euwest-cassandra-replay.name network_security_group_id = data.azurerm_network_security_group.worker-nsg.id enable_accelerated_networking = true ip_configuration { name = "vaultNicConfiguration" subnet_id = data.azurerm_subnet.default.id public_ip_address_id = "" private_ip_address_allocation = "Dynamic" } depends_on = [azurerm_resource_group.euwest-cassandra-replay] } resource "azurerm_virtual_machine" "cassandra-replay-server" { for_each = local.cassandra_replay_servers depends_on = [azurerm_resource_group.euwest-cassandra-replay] name = each.key location = "westeurope" resource_group_name = azurerm_resource_group.euwest-cassandra-replay.name network_interface_ids = [azurerm_network_interface.cassandra-replayer-interface[each.key].id] vm_size = "Standard_F8s_v2" delete_os_disk_on_termination = true delete_data_disks_on_termination = true boot_diagnostics { enabled = true storage_uri = var.boot_diagnostics_uri } storage_os_disk { name = format("%s-osdisk", each.key) caching = "ReadWrite" create_option = "FromImage" managed_disk_type = "Premium_LRS" } storage_image_reference { publisher = "debian" offer = "debian-10" sku = "10" version = "latest" } os_profile { computer_name = each.key admin_username = var.user_admin } os_profile_linux_config { disable_password_authentication = true ssh_keys { path = "/home/${var.user_admin}/.ssh/authorized_keys" key_data = var.ssh_key_data_olasd } } provisioner "remote-exec" { inline = [ "sudo mkdir /root/.ssh", "echo ${var.ssh_key_data_ardumont} | sudo tee -a /root/.ssh/authorized_keys", "echo ${var.ssh_key_data_olasd} | sudo tee -a /root/.ssh/authorized_keys", ] connection { type = "ssh" user = var.user_admin host = azurerm_network_interface.cassandra-replayer-interface[self.name].private_ip_address } } provisioner "file" { content = templatefile("templates/firstboot.sh.tpl", { hostname = self.name fqdn = format("%s.euwest.azure.internal.softwareheritage.org", self.name) ip_address = azurerm_network_interface.cassandra-replayer-interface[self.name].private_ip_address - facter_location = "azure_euwest", - disks = [] - raids = [] + facter_location = "azure_euwest" + disk_setup = {} }) destination = var.firstboot_script connection { type = "ssh" user = "root" host = azurerm_network_interface.cassandra-replayer-interface[self.name].private_ip_address } } provisioner "remote-exec" { inline = [ "userdel -f ${var.user_admin}", "chmod +x ${var.firstboot_script}", "cat ${var.firstboot_script}", "${var.firstboot_script}", ] connection { type = "ssh" user = "root" host = azurerm_network_interface.cassandra-replayer-interface[self.name].private_ip_address } } tags = { environment = "Cassandra" } } diff --git a/azure/terraform/cassandra.tf b/azure/terraform/cassandra.tf index b95c5ac..370d434 100644 --- a/azure/terraform/cassandra.tf +++ b/azure/terraform/cassandra.tf @@ -1,179 +1,181 @@ variable "cassandra_servers" { default = 6 } variable "cassandra_disk_size" { default = 1024 } variable "cassandra_disks_per_server" { default = 4 } resource "azurerm_resource_group" "euwest-cassandra" { name = "euwest-cassandra" location = "westeurope" tags = { environment = "Cassandra" } } locals { cassandra_servers = { for i in range(var.cassandra_servers): format("cassandra%02d", i + 1) => { datadisks = { for i in range(var.cassandra_disks_per_server): format("datadisk%02d", i + 1) => { lun = i + 1 path = format("/dev/disk/azure/scsi1/lun%d", i + 1) } } } } } resource "azurerm_network_interface" "cassandra-interface" { for_each = local.cassandra_servers name = format("%s-interface", each.key) location = "westeurope" resource_group_name = azurerm_resource_group.euwest-cassandra.name network_security_group_id = data.azurerm_network_security_group.worker-nsg.id enable_accelerated_networking = true ip_configuration { name = "vaultNicConfiguration" subnet_id = data.azurerm_subnet.default.id public_ip_address_id = "" private_ip_address_allocation = "Dynamic" } depends_on = [azurerm_resource_group.euwest-cassandra] } resource "azurerm_virtual_machine" "cassandra-server" { for_each = local.cassandra_servers depends_on = [azurerm_resource_group.euwest-cassandra] name = each.key location = "westeurope" resource_group_name = azurerm_resource_group.euwest-cassandra.name network_interface_ids = [azurerm_network_interface.cassandra-interface[each.key].id] vm_size = "Standard_DS13_v2" delete_os_disk_on_termination = true delete_data_disks_on_termination = true boot_diagnostics { enabled = true storage_uri = var.boot_diagnostics_uri } storage_os_disk { name = format("%s-osdisk", each.key) caching = "ReadWrite" create_option = "FromImage" managed_disk_type = "Premium_LRS" } dynamic storage_data_disk { for_each = each.value.datadisks content { name = format("%s-%s", each.key, storage_data_disk.key) caching = "None" create_option = "Empty" managed_disk_type = "Premium_LRS" disk_size_gb = var.cassandra_disk_size lun = storage_data_disk.value.lun } } storage_image_reference { publisher = "debian" offer = "debian-10" sku = "10" version = "latest" } os_profile { computer_name = each.key admin_username = var.user_admin } os_profile_linux_config { disable_password_authentication = true ssh_keys { path = "/home/${var.user_admin}/.ssh/authorized_keys" key_data = var.ssh_key_data_olasd } } provisioner "remote-exec" { inline = [ "sudo mkdir /root/.ssh", "echo ${var.ssh_key_data_ardumont} | sudo tee -a /root/.ssh/authorized_keys", "echo ${var.ssh_key_data_olasd} | sudo tee -a /root/.ssh/authorized_keys", ] connection { type = "ssh" user = var.user_admin host = azurerm_network_interface.cassandra-interface[self.name].private_ip_address } } provisioner "file" { content = templatefile("templates/firstboot.sh.tpl", { hostname = self.name fqdn = format("%s.euwest.azure.internal.softwareheritage.org", self.name) ip_address = azurerm_network_interface.cassandra-interface[self.name].private_ip_address - facter_location = "azure_euwest", + facter_location = "azure_euwest" + disk_setup = { disks = [ for disk in local.cassandra_servers[self.name].datadisks: { base_disk = disk.path } ] raids = [{ path = "/dev/md0" level = 0 chunk = "128K" members = [for disk in local.cassandra_servers[self.name].datadisks: format("%s-part1", disk.path)] mountpoint = "/srv/cassandra" filesystem = "ext4" mount_options = "defaults" }] + } }) destination = var.firstboot_script connection { type = "ssh" user = "root" host = azurerm_network_interface.cassandra-interface[self.name].private_ip_address } } provisioner "remote-exec" { inline = [ "userdel -f ${var.user_admin}", "chmod +x ${var.firstboot_script}", "cat ${var.firstboot_script}", "${var.firstboot_script}", ] connection { type = "ssh" user = "root" host = azurerm_network_interface.cassandra-interface[self.name].private_ip_address } } tags = { environment = "Cassandra" } } diff --git a/azure/terraform/kafka.tf b/azure/terraform/kafka.tf index 3945908..d3d58a1 100644 --- a/azure/terraform/kafka.tf +++ b/azure/terraform/kafka.tf @@ -1,179 +1,180 @@ variable "kafka_servers" { default = 6 } variable "kafka_disk_size" { default = 8192 } resource "azurerm_resource_group" "euwest-kafka" { name = "euwest-kafka" location = "westeurope" tags = { environment = "Kafka" } } resource "azurerm_network_security_group" "kafka-public-nsg" { name = "kafka-public-nsg" location = "westeurope" resource_group_name = "euwest-kafka" security_rule { name = "kafka-icmp-inbound-public" priority = 1000 direction = "Inbound" access = "Allow" protocol = "Icmp" source_port_range = "*" destination_port_range = "*" source_address_prefix = "*" destination_address_prefix = "VirtualNetwork" } security_rule { name = "kafka-tls-inbound-public" priority = 2000 direction = "Inbound" access = "Allow" protocol = "Tcp" source_port_range = "*" destination_port_range = "9093" source_address_prefix = "*" destination_address_prefix = "VirtualNetwork" } } resource "azurerm_public_ip" "kafka-public-ip" { count = var.kafka_servers name = format("kafka%02d-ip", count.index + 1) domain_name_label = format("swh-kafka%02d", count.index + 1) location = "westeurope" resource_group_name = "euwest-kafka" allocation_method = "Static" sku = "Standard" } resource "azurerm_network_interface" "kafka-interface" { count = var.kafka_servers name = format("kafka%02d-interface", count.index + 1) location = "westeurope" resource_group_name = "euwest-kafka" network_security_group_id = azurerm_network_security_group.kafka-public-nsg.id ip_configuration { name = "vaultNicConfiguration" subnet_id = data.azurerm_subnet.default.id public_ip_address_id = azurerm_public_ip.kafka-public-ip[count.index].id private_ip_address_allocation = "Dynamic" } } resource "azurerm_virtual_machine" "kafka-server" { count = var.kafka_servers name = format("kafka%02d", count.index + 1) location = "westeurope" resource_group_name = "euwest-kafka" network_interface_ids = [azurerm_network_interface.kafka-interface[count.index].id] vm_size = "Standard_B2s" boot_diagnostics { enabled = true storage_uri = var.boot_diagnostics_uri } storage_os_disk { name = format("kafka%02d-osdisk", count.index + 1) caching = "ReadWrite" create_option = "FromImage" managed_disk_type = "Premium_LRS" } storage_data_disk { name = format("kafka%02d-datadisk", count.index + 1) caching = "None" create_option = "Empty" managed_disk_type = "Standard_LRS" disk_size_gb = var.kafka_disk_size lun = 1 } storage_image_reference { publisher = "credativ" offer = "Debian" sku = "9" version = "latest" } os_profile { computer_name = format("kafka%02d", count.index + 1) admin_username = var.user_admin } os_profile_linux_config { disable_password_authentication = true ssh_keys { path = "/home/${var.user_admin}/.ssh/authorized_keys" key_data = var.ssh_key_data_olasd } } provisioner "remote-exec" { inline = [ "sudo mkdir /root/.ssh", "echo ${var.ssh_key_data_ardumont} | sudo tee -a /root/.ssh/authorized_keys", "echo ${var.ssh_key_data_olasd} | sudo tee -a /root/.ssh/authorized_keys", ] connection { type = "ssh" user = var.user_admin host = azurerm_network_interface.kafka-interface[count.index].private_ip_address } } provisioner "file" { content = templatefile("templates/firstboot.sh.tpl", { hostname = format("kafka%02d", count.index + 1), fqdn = format("kafka%02d.euwest.azure.internal.softwareheritage.org", count.index + 1), ip_address = azurerm_network_interface.kafka-interface[count.index].private_ip_address, facter_location = "azure_euwest", + disk_setup = { disks = [{ base_disk = "/dev/sdc", mountpoint = "/srv/kafka", filesystem = "ext4", mount_options = "defaults", }] - raids = [] + } }) destination = var.firstboot_script connection { type = "ssh" user = "root" host = azurerm_network_interface.kafka-interface[count.index].private_ip_address } } provisioner "remote-exec" { inline = [ "userdel -f ${var.user_admin}", "chmod +x ${var.firstboot_script}", "cat ${var.firstboot_script}", "${var.firstboot_script}", ] connection { type = "ssh" user = "root" host = azurerm_network_interface.kafka-interface[count.index].private_ip_address } } tags = { environment = "Kafka" } } diff --git a/azure/terraform/storage.tf b/azure/terraform/storage.tf index f244d86..1400b58 100644 --- a/azure/terraform/storage.tf +++ b/azure/terraform/storage.tf @@ -1,131 +1,130 @@ # will start from 1 storage01... variable "storage_servers" { default = 2 } variable "storage_disk_size" { default = 30720 } locals { storage_servers = { for i in range(var.storage_servers): format("storage%02d", i + 1) => { datadisks = {} } } } resource "azurerm_network_interface" "storage-interface" { for_each = local.storage_servers name = format("%s-interface", each.key) location = "westeurope" resource_group_name = "euwest-servers" network_security_group_id = data.azurerm_network_security_group.worker-nsg.id ip_configuration { name = "storageNicConfiguration" subnet_id = data.azurerm_subnet.default.id public_ip_address_id = "" private_ip_address_allocation = "Dynamic" } } resource "azurerm_virtual_machine" "storage-server" { for_each = local.storage_servers name = each.key location = "westeurope" resource_group_name = "euwest-servers" network_interface_ids = [azurerm_network_interface.storage-interface[each.key].id] vm_size = "Standard_D8s_v3" boot_diagnostics { enabled = true storage_uri = var.boot_diagnostics_uri } storage_os_disk { name = format("%s-osdisk", each.key) caching = "ReadWrite" create_option = "FromImage" managed_disk_type = "Premium_LRS" } storage_image_reference { publisher = "debian" offer = "debian-10" sku = "10" version = "latest" } os_profile { computer_name = each.key admin_username = var.user_admin } os_profile_linux_config { disable_password_authentication = true ssh_keys { path = "/home/${var.user_admin}/.ssh/authorized_keys" key_data = var.ssh_key_data_ardumont } ssh_keys { path = "/home/${var.user_admin}/.ssh/authorized_keys" key_data = var.ssh_key_data_olasd } } provisioner "remote-exec" { inline = [ "sudo mkdir /root/.ssh", "echo ${var.ssh_key_data_ardumont} | sudo tee -a /root/.ssh/authorized_keys", "echo ${var.ssh_key_data_olasd} | sudo tee -a /root/.ssh/authorized_keys", ] connection { type = "ssh" user = var.user_admin host = azurerm_network_interface.storage-interface[each.key].private_ip_address } } provisioner "file" { content = templatefile("templates/firstboot.sh.tpl", { hostname = each.key fqdn = format("%s.euwest.azure.internal.softwareheritage.org", each.key), ip_address = azurerm_network_interface.storage-interface[each.key].private_ip_address, - facter_location = "azure_euwest", - disks = [] - raids = [] + facter_location = "azure_euwest" + disk_setup = {} }) destination = var.firstboot_script connection { type = "ssh" user = "root" host = azurerm_network_interface.storage-interface[each.key].private_ip_address } } provisioner "remote-exec" { inline = [ "userdel -f ${var.user_admin}", "chmod +x ${var.firstboot_script}", "cat ${var.firstboot_script}", "${var.firstboot_script}", ] connection { type = "ssh" user = "root" host = azurerm_network_interface.storage-interface[each.key].private_ip_address } } tags = { environment = "Storage" } } diff --git a/azure/terraform/templates/firstboot.sh.tpl b/azure/terraform/templates/firstboot.sh.tpl index 9a2cb9b..64fb472 100644 --- a/azure/terraform/templates/firstboot.sh.tpl +++ b/azure/terraform/templates/firstboot.sh.tpl @@ -1,102 +1,129 @@ #!/bin/bash set -ex cd / PUPPET_MASTER=pergamon.internal.softwareheritage.org # Variables provided by terraform HOSTNAME=${hostname} FQDN=${fqdn} IP=${ip_address} FACTER_LOCATION=${facter_location} # Handle base system configuration apt-get -y install lsb-release debian_suite=$(lsb_release -cs) # Enable backports cat > /etc/apt/sources.list.d/backports.list < /etc/hostname hostnamectl set-hostname $HOSTNAME echo "$IP $FQDN $HOSTNAME" >> /etc/hosts # Handle disk configuration -%{ for disk in disks } +%{ for disk in try(disk_setup.disks, []) } # Make one large partition on ${disk.base_disk} echo ';' | sudo sfdisk --label gpt ${disk.base_disk} -%{ if lookup(disk, "filesystem", "") != "" } -mkfs.${disk.filesystem} ${disk.base_disk}1 +%{ if try(disk.filesystem, "") != "" } +mkfs.${disk.filesystem} ${disk.base_disk}1 ${try(disk.mkfs_options, "")} mkdir -p ${disk.mountpoint} uuid=$(blkid -o value -s UUID ${disk.base_disk}1) echo "UUID=\"$uuid\" ${disk.mountpoint} ${disk.filesystem} ${disk.mount_options} 0 0" >> /etc/fstab %{ endif } %{ endfor } -%{ if length(raids) != 0 } +%{ if length(try(disk_setup.raids, [])) != 0 } apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --no-install-recommends install mdadm -%{ for raid in raids } +%{ for raid in disk_setup.raids } mdadm --create ${raid.path} \ --level=${raid.level} \ --raid-devices ${length(raid.members)} \ %{ if raid.chunk != "" }--chunk=${raid.chunk}%{ endif } \ %{~ for member in raid.members } ${member} %{ endfor ~} -%{ if lookup(raid, "filesystem", "") != "" } -mkfs.${raid.filesystem} ${raid.path} +%{ if try(raid.filesystem, "") != "" } +mkfs.${raid.filesystem} ${raid.path} ${try(raid.mkfs_options, "")} mkdir -p ${raid.mountpoint} uuid=$(blkid -o value -s UUID ${raid.path}) echo "UUID=\"$uuid\" ${raid.mountpoint} ${raid.filesystem} ${raid.mount_options} 0 0" >> /etc/fstab %{ endif } %{ endfor } /usr/share/mdadm/mkconf > /etc/mdadm/mdadm.conf update-initramfs -k all -u %{ endif } +%{ if length(try(disk_setup.lvm_vgs, [])) != 0 } + +apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --no-install-recommends install lvm2 + +%{ for lvm_vg in disk_setup.lvm_vgs } + +vgcreate ${lvm_vg.name} ${join(" ", lvm_vg.pvs)} + +%{ for lvm_lv in lvm_vg.lvs } + +lvcreate ${lvm_vg.name} -n ${lvm_lv.name} -l ${lvm_lv.extents} + +%{ if try(lvm_lv.filesystem, "") != "" } +mkfs.${lvm_lv.filesystem} /dev/${lvm_vg.name}/${lvm_lv.name} ${try(lvm_lv.mkfs_options, "")} + +mkdir -p ${lvm_lv.mountpoint} + +uuid=$(blkid -o value -s UUID /dev/${lvm_vg.name}/${lvm_lv.name}) +echo "UUID=\"$uuid\" ${lvm_lv.mountpoint} ${lvm_lv.filesystem} ${lvm_lv.mount_options} 0 0" >> /etc/fstab +%{ endif } +%{ endfor } +%{ endfor } + +update-initramfs -k all -u +%{ endif } + + mount -a # install puppet dependencies apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --no-install-recommends install puppet gnupg # do not need the service live as we need to install some more setup first service puppet stop systemctl disable puppet.service # Install the location fact as that is needed by our puppet manifests mkdir -p /etc/facter/facts.d echo location=$FACTER_LOCATION > /etc/facter/facts.d/location.txt # first time around, this will: # - update the node's puppet agent configuration defining the puppet master # - generate the certificates with the appropriate fqdn puppet_exit=0 puppet agent --server $PUPPET_MASTER --waitforcert 60 --test --vardir /var/lib/puppet --detailed-exitcodes || puppet_exit=$? if [ $puppet_exit -ne 2 ]; then exit $puppet_exit fi # reboot