diff --git a/sysadmin/grid5000/cassandra/Vagrantfile b/sysadmin/grid5000/cassandra/Vagrantfile index 5ff3003..ee1bf9e 100644 --- a/sysadmin/grid5000/cassandra/Vagrantfile +++ b/sysadmin/grid5000/cassandra/Vagrantfile @@ -1,60 +1,69 @@ # -*- mode: ruby -*- # vi: set ft=ruby : vms = { "cassandra1" => { :ip => "10.168.180.11", :memory => 2048, :cpus => 2, + :type => 'cassandra', }, "cassandra2" => { :ip => "10.168.180.12", :memory => 2048, :cpus => 2, + :type => 'cassandra', }, "cassandra3" => { :ip => "10.168.180.13", :memory => 2048, :cpus => 2, + :type => 'cassandra', }, } # Images/remote configuration $global_debian10_box = "debian10-20210517-1348" $global_debian10_box_url = "https://annex.softwareheritage.org/public/isos/libvirt/debian/swh-debian-10.9-amd64-20210517-1348.qcow2" vms.each { | vm_name, vm_props | Vagrant.configure("2") do |global_config| unless Vagrant.has_plugin?("libvirt") $stderr.puts <<-MSG vagrant-libvirt plugin is required for this. To install: `$ sudo apt install vagrant-libvirt MSG exit 1 end global_config.vm.define vm_name do |config| config.vm.box = $global_debian10_box config.vm.box_url = $global_debian10_box_url config.vm.box_check_update = false config.vm.hostname = vm_name config.vm.network :private_network, ip: vm_props[:ip], netmask: "255.255.0.0" - config.vm.synced_folder ".", "/vagrant", type: 'nfs' + config.vm.synced_folder ".", "/vagrant", type: 'nfs', nfs_version: 4 config.vm.provision :ansible do |ansible| ansible.verbose = true ansible.become = true ansible.playbook = "ansible/playbook.yml" ansible.inventory_path = "ansible/hosts.yml" end config.vm.provider :libvirt do |provider| provider.memory = vm_props[:memory] provider.cpus = vm_props[:cpus] provider.driver = 'kvm' + if vm_props[:type] == "cassandra" + provider.storage :file, :size => '1G' + provider.storage :file, :size => '1G' + provider.storage :file, :size => '1G' + end + end end end } diff --git a/sysadmin/grid5000/cassandra/ansible/_zfs_create_dataset.yml b/sysadmin/grid5000/cassandra/ansible/_zfs_create_dataset.yml new file mode 100644 index 0000000..8320e92 --- /dev/null +++ b/sysadmin/grid5000/cassandra/ansible/_zfs_create_dataset.yml @@ -0,0 +1,15 @@ +--- + +- name: Prepare datasets + set_fact: + datasets: "{{ datasets | default({}) | combine ({ [obj.key, item.key] | join('/') : item.value }) }}" + loop: "{{ lookup('dict', obj.value.datasets, wantlist=True) }}" + +- name: Create zfs dataset + shell: "zfs create -o mountpoint={{item.value}} {{item.key}}" + loop: "{{ lookup('dict', datasets, wantlist=True) }}" + when: item.key not in pool_list.stdout_lines + +- name: Clean local variable + set_fact: + datasets: {} diff --git a/sysadmin/grid5000/cassandra/ansible/cassandra.yml b/sysadmin/grid5000/cassandra/ansible/cassandra.yml new file mode 100644 index 0000000..94930c6 --- /dev/null +++ b/sysadmin/grid5000/cassandra/ansible/cassandra.yml @@ -0,0 +1,57 @@ +--- +# - name: "Get public ipv4 address" +# set_fact: +# cassandra_seed_ips: "{{ansible_facts[item]['ipv4']['address']}}" +# with_items: +# - "{{cassandra_listen_interface }}" + +- name: Install cassandra signing key + apt_key: + url: https://downloads.apache.org/cassandra/KEYS + state: present + +- name: Install cassandra apt repository + apt_repository: + repo: deb http://downloads.apache.org/cassandra/debian 40x main + state: present + filename: cassandra.sources + +- name: Install cassandra packages + apt: + update_cache: true # force an apt update before + name: + ## TODO: check other version than jdk11 + - openjdk-11-jdk + - cassandra + - dstat + - smartmontools + +- name: Create datadirs + file: + state: directory + path: "{{ item }}" + owner: "cassandra" + group: "cassandra" + mode: "0755" + with_items: + - "{{ cassandra_data_dir_base }}" + - "{{ cassandra_data_dir_system }}" + - "{{ cassandra_data_dir }}" + - "{{ cassandra_commitlogs_dir }}" + +- name: Configure cassandra + template: + src: "templates/{{item}}" + dest: "{{cassandra_config_dir}}/{{item}}" + with_items: [cassandra.yaml, jvm.options] + register: cassandra_configuration_files + +- name: Restart cassandra service + service: + name: cassandra + state: restarted + when: cassandra_configuration_files.changed + + + + # TODO test different read ahead diff --git a/sysadmin/grid5000/cassandra/ansible/hosts.yml b/sysadmin/grid5000/cassandra/ansible/hosts.yml index 5c4de68..f5797af 100644 --- a/sysadmin/grid5000/cassandra/ansible/hosts.yml +++ b/sysadmin/grid5000/cassandra/ansible/hosts.yml @@ -1,36 +1,76 @@ +# Global configuration cassandra: hosts: - dahu[1:32].grenoble.grid5000.fr: + dahu-[1:32].grenoble.grid5000.fr: + parasilo-[2:4].rennes.grid5000.fr: # local vagrant hosts cassandra[1:9]: vars: + ansible_connection: local cassandra_config_dir: /etc/cassandra cassandra_data_dir_base: /srv/cassandra cassandra_data_dir_system: "{{cassandra_data_dir_base}}/system" cassandra_data_dir: "{{ cassandra_data_dir_base }}/data" cassandra_commitlogs_dir: "{{ cassandra_data_dir_base }}/commitlogs" + +# Per cluster specificities +dahu_cluster_hosts: + hosts: + dahu[1:32].grenoble.grid5000.fr + vars: + cassandra_listen_interface: enp24s0f0 + +parasilo_cluster_hosts: + hosts: + parasilo-[1:28].rennes.grid5000.fr: + vars: + cassandra_listen_interface: eno1 + zfs_pools: + commitlogs: + disks: + - sdf + datasets: + commitlogs: /srv/cassandra/commitlogs + data: + disks: + - sdb + - sdc + - sdd + - sde + datasets: + data: /srv/cassandra/data + + +# Vagrant configuration vagrant_nodes: hosts: cassandra1: ansible_host: 10.168.180.11 ansible_user: vagrant ansible_ssh_private_key_file: .vagrant/machines/cassandra1/libvirt/private_key cassandra2: ansible_host: 10.168.180.12 ansible_user: vagrant ansible_ssh_private_key_file: .vagrant/machines/cassandra2/libvirt/private_key cassandra3: ansible_host: 10.168.180.13 ansible_user: vagrant ansible_ssh_private_key_file: .vagrant/machines/cassandra2/libvirt/private_key vars: + ansible_connection: ssh cassandra_listen_interface: eth1 # passed through --extra-vars on grid5000 cassandra_seed_ips: 10.168.180.11,10.168.180.12,10.168.180.13 - -dahu_cluster_hosts: - hosts: - dahu[1:32].grenoble.grid5000.fr - vars: - cassandra_listen_interface: enp24s0f0 + zfs_pools: + commitlogs: + disks: + - vdb + datasets: + commitlogs: /srv/cassandra/commitlogs + data: + disks: + - vdc + - vdd + datasets: + data: /srv/cassandra/data diff --git a/sysadmin/grid5000/cassandra/ansible/playbook.yml b/sysadmin/grid5000/cassandra/ansible/playbook.yml index b5a58a7..4bc7e98 100644 --- a/sysadmin/grid5000/cassandra/ansible/playbook.yml +++ b/sysadmin/grid5000/cassandra/ansible/playbook.yml @@ -1,60 +1,6 @@ --- - name: Install cassandra hosts: cassandra - tasks: - # - name: "Get public ipv4 address" - # set_fact: - # cassandra_seed_ips: "{{ansible_facts[item]['ipv4']['address']}}" - # with_items: - # - "{{cassandra_listen_interface }}" - - - name: Install cassandra signing key - apt_key: - url: https://downloads.apache.org/cassandra/KEYS - state: present - - - name: Install cassandra apt repository - apt_repository: - repo: deb http://downloads.apache.org/cassandra/debian 40x main - state: present - filename: cassandra.sources.list - - - name: Install packages - apt: - update_cache: true # force an apt update before - name: - ## TODO: check other version than jdk11 - - openjdk-11-jdk - - cassandra - - dstat - - - name: Create datadirs - file: - state: directory - path: "{{ item }}" - owner: "cassandra" - group: "cassandra" - mode: "0755" - with_items: - - "{{ cassandra_data_dir_base }}" - - "{{ cassandra_data_dir_system }}" - - "{{ cassandra_data_dir }}" - - "{{ cassandra_commitlogs_dir }}" - - - name: Configure cassandra - template: - src: "templates/{{item}}" - dest: "{{cassandra_config_dir}}/{{item}}" - with_items: [cassandra.yaml, jvm.options] - register: cassandra_configuration_files - - - name: Restart cassandra service - service: - name: cassandra - state: restarted - when: cassandra_configuration_files.changed - - - - # TODO test different read ahead + - include: zfs.yml + - include: cassandra.yml diff --git a/sysadmin/grid5000/cassandra/ansible/templates/cassandra.yaml b/sysadmin/grid5000/cassandra/ansible/templates/cassandra.yaml index 7511ccc..7a92e70 100644 --- a/sysadmin/grid5000/cassandra/ansible/templates/cassandra.yaml +++ b/sysadmin/grid5000/cassandra/ansible/templates/cassandra.yaml @@ -1,37 +1,37 @@ cluster_name: swh-storage # default 'Test Cluster' num_tokens: 256 # default 256 allocate_tokens_for_local_replication_factor: 3 data_file_directories: - {{ cassandra_data_dir }} # TODO use several disks -local_system_data_file_directory: {{ cassandra_data_dir_system }} +# local_system_data_file_directory: {{ cassandra_data_dir_system }} commitlog_directory: {{ cassandra_commitlogs_dir }} disk_optimization_strategy: spinning # spinning | ssd # listen_address: 0.0.0.0 # always wrong according to the documentation listen_interface: {{ cassandra_listen_interface }} # always wrong according to the documentation concurrent_compactors: 1 # should be min(nb core, nb disks) internode_compression: dc # default dc possible all|dc|none concurrent_reads: 16 # 16 x number of drives concurrent_writes: 32 # 8 x number of cores commitlog_sync: periodic # default periodic commitlog_sync_period_in_ms: 10000 # default 10000 partitioner: org.apache.cassandra.dht.Murmur3Partitioner endpoint_snitch: SimpleSnitch seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: # seeds is actually a comma-delimited list of addresses. # Ex: ",," - seeds: "{{ cassandra_seed_ips }}" # TODO Test this options effects # disk_failure_policy: # cdc_enabled #end diff --git a/sysadmin/grid5000/cassandra/ansible/zfs.yml b/sysadmin/grid5000/cassandra/ansible/zfs.yml new file mode 100644 index 0000000..5f481b4 --- /dev/null +++ b/sysadmin/grid5000/cassandra/ansible/zfs.yml @@ -0,0 +1,76 @@ +--- +- name: Install contrib and non-free repositories + apt_repository: + repo: deb http://deb.debian.org/debian/ buster-backports main contrib non-free + filename: backports.sources + +- name: Install zfs packages + apt: + update_cache: true # force an apt update before + name: + - linux-image-amd64 + - linux-headers-amd64 + - libnvpair1linux + - libuutil1linux + - libzfs2linux + - libzpool2linux + - zfs-dkms + - zfs-zed + - zfsutils-linux + ignore_errors: True + +- name: Ensure zfs initialized + shell: /usr/sbin/modprobe zfs + +- name: Finalize zfs packages installation + apt: + update_cache: true # force an apt update before + name: + - linux-image-amd64 + - linux-headers-amd64 + - libnvpair1linux + - libuutil1linux + - libzfs2linux + - libzpool2linux + - zfs-dkms + - zfs-zed + - zfsutils-linux + +- name: Install zfs packages after modprobe + apt: + update_cache: true # force an apt update before + name: + - zfsutils-linux + - zfs-zed + +- name: test if zfs pools are configured + command: zfs list -o name + register: pool_list + +- name: zfs pool import + shell: "zpool import {{ item.key }}" + loop: "{{ lookup('dict', zfs_pools) }}" + when: item.key not in pool_list.stdout_lines + ignore_errors: True + +- name: test if zfs pools are configured + command: zfs list -o name + register: pool_list + +- name: create zfs pools + shell: "zpool create -f {{ item.key }} {{ item.value.disks | join(' ') }}" + loop: "{{ lookup('dict', zfs_pools) }}" + when: item.key not in pool_list.stdout_lines + +- name: call dataset creation + include: _zfs_create_dataset.yml obj={{ outside_item }} + loop: "{{ lookup('dict', zfs_pools) }}" + loop_control: + loop_var: outside_item + +- name: call dataset creation + include: _zfs_create_dataset.yml obj={{ outside_item }} + loop: "{{ lookup('dict', zfs_pools) }}" + loop_control: + loop_var: outside_item + when: outside_item.key not in pool_list.stdout_lines