root@kelvingrove:~# task=T3807
root@kelvingrove:/etc# puppet agent --disable "$task: dist-upgrade to bullseye"
root@kelvingrove:/etc# sed -i -e 's/buster/bullseye/;s,bullseye/updates,bullseye-security,' /etc/apt/sources.list.d/*

	Correctable memory error logging disabled for a memory device at location DIMM_A9. 	Fri 17 Dec 2021 16:15:39

root@moma:/etc/softwareheritage/storage# puppet agent --disable 'T3801 upgrade database servers'

root@e35f7a024575:/home/jenkins/swh-environment/swh-indexer# gdb python3 core
(gdb) where
#0  raise (sig=11) at ../sysdeps/unix/sysv/linux/raise.c:50
#1  <signal handler called>
#2  0x00007f6548d70d46 in frozendict_new_barebone (type=0x7f6548d800e0 <PyFrozenDict_Type>)
    at /project/frozendict/src/3_7/frozendictobject.c:2214
#3  _frozendict_new (use_empty_frozendict=1, kwds=0x0, args=<optimized out>, type=0x7f6548d800e0 <PyFrozenDict_Type>)
    at /project/frozendict/src/3_7/frozendictobject.c:2255
#4  frozendict_new (type=0x7f6548d800e0 <PyFrozenDict_Type>, args=<optimized out>, kwds=0x0)
    at /project/frozendict/src/3_7/frozendictobject.c:2290
#5  0x00000000005d9bd7 in _PyObject_FastCallKeywords ()
#136 0x000000000065468e in _Py_UnixMain ()
#137 0x00007f654efe109b in __libc_start_main (main=0x4bc560 <main>, argc=9, argv=0x7ffe6f651488, init=<optimized out>, 
    fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffe6f651478) at ../csu/libc-start.c:308
#138 0x00000000005e0e8a in _start ()
(gdb)

root@pergamon:/etc/clustershell# clush -b -w @staging 'if [ -e /etc/systemd/system/cloud-init.target.wants/cloud-init.service ]; then echo "cloud-init installed"; echo cloudinit_enabled=true > /etc/facter/facts.d/cloud-init.txt; else echo "cloud-init not installed"; fi'
---------------
counters0.internal.staging.swh.network,deposit.internal.staging.swh.network,objstorage0.internal.staging.swh.network,poc-rancher-sw[0-1].internal.staging.swh.network,poc-rancher.internal.staging.swh.network,rp0.internal.staging.swh.network,scheduler0.internal.staging.swh.network,search0.internal.staging.swh.network,vault.internal.staging.swh.network,webapp.internal.staging.swh.network,worker[0-3].internal.staging.swh.network (15)
---------------
cloud-init installed
---------------
db1.internal.staging.swh.network,storage1.internal.staging.swh.network (2)
---------------
cloud-init not installed

root@pergamon:~# clush -b -w @staging-workers 'set -e; puppet agent --disable "T3812"; puppet agent --disable T3771; cd /etc/systemd/system/multi-user.target.wants; for unit in swh-worker@*; do systemctl disable $unit; done; systemctl stop --no-block swh-worker@*; sleep 300; systemctl kill swh-worker@* -s 9'

#cloud-config-jsonp
[{ "op": "replace", "path": "/manage_etc_hosts", "value": "False"}]

2021-12-16 22:35:11,471 - __init__.py[DEBUG]: Calling handler CloudConfigPartHandler: [['text/cloud-config', 'text/cloud-config-jsonp']] (text/cloud-config-jsonp, part-001, 3) with frequency always
2021-12-16 22:35:11,472 - cloud_config.py[DEBUG]: Merging by applying json patch [{"op": "replace", "path": "/manage_etc_hosts", "value": "False"}]
2021-12-16 22:35:11,472 - util.py[WARNING]: Failed at merging in cloud config part from part-001
2021-12-16 22:35:11,474 - util.py[DEBUG]: Failed at merging in cloud config part from part-001
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/cloudinit/handlers/cloud_config.py", line 138, in handle_part
    self._merge_patch(payload)
  File "/usr/lib/python3/dist-packages/cloudinit/handlers/cloud_config.py", line 113, in _merge_patch
    self.cloud_buf = patch.apply(self.cloud_buf, in_place=False)
  File "/usr/lib/python3/dist-packages/jsonpatch.py", line 312, in apply
    obj = operation.apply(obj)
  File "/usr/lib/python3/dist-packages/jsonpatch.py", line 483, in apply
    raise JsonPatchConflict(msg)
jsonpatch.JsonPatchConflict: can't replace non-existent object 'manage_etc_hosts'
2021-12-16 22:35:11,475 - __init__.py[DEBUG]: Calling handler CloudConfigPartHandler: [['text/cloud-config', 'text/cloud-config-jsonp']] (__end__, None, 3) with frequency always

root@pergamon:~# /usr/local/sbin/swh-puppet-master-decommission boatbucket.internal.softwareheritage.org
+ puppet node deactivate boatbucket.internal.softwareheritage.org
Submitted 'deactivate node' for boatbucket.internal.softwareheritage.org with UUID bf7bd0ea-f1ae-442f-b840-5bb1adb261f3
+ puppet node clean boatbucket.internal.softwareheritage.org
Notice: Revoked certificate with serial 224
Notice: Removing file Puppet::SSL::Certificate boatbucket.internal.softwareheritage.org at '/var/lib/puppet/ssl/ca/signed/boatbucket.internal.softwareheritage.org.pem'
boatbucket.internal.softwareheritage.org
+ puppet cert clean boatbucket.internal.softwareheritage.org
Warning: `puppet cert` is deprecated and will be removed in a future release.
   (location: /usr/lib/ruby/vendor_ruby/puppet/application.rb:370:in `run')
Notice: Revoked certificate with serial 224
+ systemctl restart apache2
root@pergamon:~# puppet agent --test

root@boatbucket:/home# ls -d alphare/* alphare/.bash_history boatbucket/* boatbucket/.bash_history | xargs tar cvjf boatbucket-backup-2021-12-14.tar.bz2
...

root@boatbucket:/home# sudo -u boatbucket cp -v boatbucket-backup-2021-12-14.tar.bz /srv/boatbucket/
'boatbucket-backup-2021-12-14.tar.bz' -> '/srv/boatbucket/boatbucket-backup-2021-12-14.tar.bz'
root@boatbucket:/home# ls -al /srv/boatbucket/boatbucket-backup-2021-12-14.tar.bz 
-rw-r--r-- 1 boatbucket boatbucket 124170240 Dec 14 15:36 /srv/boatbucket/boatbucket-backup-2021-12-14.tar.bz
root@boatbucket:/home# mount | grep boatbucket
systemd-1 on /srv/boatbucket type autofs (rw,relatime,fd=54,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13290)
saam:/srv/storage/space/mirrors/boatbucket on /srv/boatbucket type nfs4 (rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,soft,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.100.107,local_lock=none,addr=192.168.100.109)