From dcb2ca053c0e82bec4e9021146c079e4aab53e0f Mon Sep 17 00:00:00 2001
From: Erik Stambaugh <erik@erikstambaugh.com>
Date: Mon, 12 Feb 2024 20:41:20 -0800
Subject: [PATCH] Backups!  Get it all into S3 in case of an emergency

---
 ansible/Makefile                           |  4 +--
 ansible/inventory.tmpl.yaml                |  1 +
 ansible/roles/common/tasks/main.yaml       |  1 +
 ansible/roles/mastodon/tasks/main.yaml     | 11 ++++++++
 ansible/roles/mastodon/templates/backup.sh | 31 ++++++++++++++++++++++
 terraform/outputs.tf                       |  3 +++
 6 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 ansible/roles/mastodon/templates/backup.sh

diff --git a/ansible/Makefile b/ansible/Makefile
index eccb62d..0e736e3 100644
--- a/ansible/Makefile
+++ b/ansible/Makefile
@@ -23,7 +23,7 @@ reboot: inventory.yaml
 	aws --region $(AWS_REGION) ec2 reboot-instances --instance-ids $(INSTANCE_ID)
 
 inventory.yaml: inventory.tmpl.yaml sedline
-	@sed $(SEDLINE) inventory.tmpl.yaml > inventory.yaml
+	sed $(SEDLINE) inventory.tmpl.yaml > inventory.yaml
 
 SEDLINE =
 
@@ -39,7 +39,7 @@ __sed_%:
 # FIXME: this is awful because it's all in the clear
 secret_sedline:
 	$(eval SEDLINE := $$(SEDLINE) -e 's/{{S3_IAM_ID}}/$(shell     head -1 ../.s3_iam_credentials)/')
-	$(eval SEDLINE := $$(SEDLINE) -e 's/{{S3_IAM_SECRET}}/$(shell tail -1 ../.s3_iam_credentials)/')
+	$(eval SEDLINE := $$(SEDLINE) -e 's/{{S3_IAM_SECRET}}/$(shell tail -1 ../.s3_iam_credentials | sed -e "s/\//\\\\\//g")/')
 	$(eval SEDLINE := $$(SEDLINE) -e 's/{{SES_IAM_ID}}/$(shell     head -1 ../.ses_iam_credentials)/')
 	$(eval SEDLINE := $$(SEDLINE) -e 's/{{SES_IAM_SECRET}}/$(shell ./ses_credentials.py `tail -1 ../.ses_iam_credentials` $(AWS_REGION) | sed -e "s/\//\\\\\//g")/')
 
diff --git a/ansible/inventory.tmpl.yaml b/ansible/inventory.tmpl.yaml
index 4a8f4e3..71a55da 100644
--- a/ansible/inventory.tmpl.yaml
+++ b/ansible/inventory.tmpl.yaml
@@ -16,6 +16,7 @@ social:
     public_ip: "{{PUBLIC_IP}}"
     my_ip: "{{MY_IP}}"
     s3_bucket_name: "{{S3_BUCKET_NAME}}"
+    s3_private_bucket_name: "{{S3_PRIVATE_BUCKET_NAME}}"
     #s3_endpoint: 
     s3_hostname: "s3.{{AWS_REGION}}.amazonaws.com"
     s3_iam_id: {{S3_IAM_ID}}
diff --git a/ansible/roles/common/tasks/main.yaml b/ansible/roles/common/tasks/main.yaml
index 7f5a1e3..2556315 100644
--- a/ansible/roles/common/tasks/main.yaml
+++ b/ansible/roles/common/tasks/main.yaml
@@ -48,6 +48,7 @@
       - ntp
       - lsof
       - net-tools
+      - awscli
 
 - name: edit bashrc
   blockinfile:
diff --git a/ansible/roles/mastodon/tasks/main.yaml b/ansible/roles/mastodon/tasks/main.yaml
index 93a53a1..643d6e1 100644
--- a/ansible/roles/mastodon/tasks/main.yaml
+++ b/ansible/roles/mastodon/tasks/main.yaml
@@ -6,6 +6,7 @@
     name:
       - docker-compose-v2
       - git
+      - xz-utils
 
 - name: base path
   file:
@@ -187,6 +188,16 @@
   #when: nginxconf.changed
 
 
+## backups!
+
+- name: copy backup script
+  template:
+    src: templates/backup.sh
+    dest: /etc/cron.daily/backup-mastodon
+    mode: 0744
+
+
+
 ## ---
 
 - name: clean up docker
diff --git a/ansible/roles/mastodon/templates/backup.sh b/ansible/roles/mastodon/templates/backup.sh
new file mode 100644
index 0000000..4144b34
--- /dev/null
+++ b/ansible/roles/mastodon/templates/backup.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+BUCKET={{s3_private_bucket_name}}
+MAX_DAILY=2
+
+DATE=$(date -I)
+
+# copy the redis backup
+aws s3 cp /srv/mastodon/redis/dump.rdb s3://${BUCKET}/backup_redis/dump-${DATE}.rdb
+
+# push a postgres backup
+docker exec -t mastodon_db pg_dumpall -c -U postgres | xz | aws s3 cp - s3://${BUCKET}/backup_postgres/dump-${DATE}.sql.xz
+
+# rotate -- let's just keep 2
+NUM_REDIS=$(aws s3 ls s3://${BUCKET}/backup_redis/ | wc -l)
+NUM_POSTGRES=$(aws s3 ls s3://${BUCKET}/backup_postgres/ | wc -l)
+
+while [[ $NUM_REDIS -gt $MAX_DAILY ]]; do
+    LAST=$(aws s3 ls s3://${BUCKET}/backup_redis/ | tail -1 | awk '{ print $4 }')
+    aws s3 rm s3://${BUCKET}/backup_redis/${LAST}
+    NUM_REDIS=$((NUM_REDIS - 1))
+done
+
+while [[ $NUM_POSTGRES -gt $MAX_DAILY ]]; do
+    LAST=$(aws s3 ls s3://${BUCKET}/backup_postgres/ | tail -1 | awk '{ print $4 }')
+    aws s3 rm s3://${BUCKET}/backup_postgres/${LAST}
+    NUM_POSTGRES=$((NUM_POSTGRES - 1))
+done
+
diff --git a/terraform/outputs.tf b/terraform/outputs.tf
index e27d5ae..9d735ca 100644
--- a/terraform/outputs.tf
+++ b/terraform/outputs.tf
@@ -12,6 +12,9 @@ output "nameservers" {
 output "s3_bucket_name" {
   value = aws_s3_bucket.s3_bucket.id
 }
+output "s3_private_bucket_name" {
+  value = module.private_s3_bucket.s3_bucket_id
+}
 output "my_ip" {
   value = "${chomp(data.http.myip.response_body)}"
 }