Backups! Get it all into S3 in case of an emergency

ansible/Makefile

@@ -23,7 +23,7 @@ reboot: inventory.yaml
 	aws --region $(AWS_REGION) ec2 reboot-instances --instance-ids $(INSTANCE_ID)
 
 inventory.yaml: inventory.tmpl.yaml sedline
-	@sed $(SEDLINE) inventory.tmpl.yaml > inventory.yaml
+	sed $(SEDLINE) inventory.tmpl.yaml > inventory.yaml
 
 SEDLINE =
 
@@ -39,7 +39,7 @@ __sed_%:
 # FIXME: this is awful because it's all in the clear
 secret_sedline:
 	$(eval SEDLINE := $$(SEDLINE) -e 's/{{S3_IAM_ID}}/$(shell     head -1 ../.s3_iam_credentials)/')
-	$(eval SEDLINE := $$(SEDLINE) -e 's/{{S3_IAM_SECRET}}/$(shell tail -1 ../.s3_iam_credentials)/')
+	$(eval SEDLINE := $$(SEDLINE) -e 's/{{S3_IAM_SECRET}}/$(shell tail -1 ../.s3_iam_credentials | sed -e "s/\//\\\\\//g")/')
 	$(eval SEDLINE := $$(SEDLINE) -e 's/{{SES_IAM_ID}}/$(shell     head -1 ../.ses_iam_credentials)/')
 	$(eval SEDLINE := $$(SEDLINE) -e 's/{{SES_IAM_SECRET}}/$(shell ./ses_credentials.py `tail -1 ../.ses_iam_credentials` $(AWS_REGION) | sed -e "s/\//\\\\\//g")/')
 

ansible/inventory.tmpl.yaml

@@ -16,6 +16,7 @@ social:
     public_ip: "{{PUBLIC_IP}}"
     my_ip: "{{MY_IP}}"
     s3_bucket_name: "{{S3_BUCKET_NAME}}"
+    s3_private_bucket_name: "{{S3_PRIVATE_BUCKET_NAME}}"
     #s3_endpoint: 
     s3_hostname: "s3.{{AWS_REGION}}.amazonaws.com"
     s3_iam_id: {{S3_IAM_ID}}

ansible/roles/common/tasks/main.yaml

@@ -48,6 +48,7 @@
       - ntp
       - lsof
       - net-tools
+      - awscli
 
 - name: edit bashrc
   blockinfile:

ansible/roles/mastodon/tasks/main.yaml

@@ -6,6 +6,7 @@
     name:
       - docker-compose-v2
       - git
+      - xz-utils
 
 - name: base path
   file:
@@ -187,6 +188,16 @@
   #when: nginxconf.changed
 
 
+## backups!
+
+- name: copy backup script
+  template:
+    src: templates/backup.sh
+    dest: /etc/cron.daily/backup-mastodon
+    mode: 0744
+
+
+
 ## ---
 
 - name: clean up docker

ansible/roles/mastodon/templates/backup.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+BUCKET={{s3_private_bucket_name}}
+MAX_DAILY=2
+
+DATE=$(date -I)
+
+# copy the redis backup
+aws s3 cp /srv/mastodon/redis/dump.rdb s3://${BUCKET}/backup_redis/dump-${DATE}.rdb
+
+# push a postgres backup
+docker exec -t mastodon_db pg_dumpall -c -U postgres | xz | aws s3 cp - s3://${BUCKET}/backup_postgres/dump-${DATE}.sql.xz
+
+# rotate -- let's just keep 2
+NUM_REDIS=$(aws s3 ls s3://${BUCKET}/backup_redis/ | wc -l)
+NUM_POSTGRES=$(aws s3 ls s3://${BUCKET}/backup_postgres/ | wc -l)
+
+while [[ $NUM_REDIS -gt $MAX_DAILY ]]; do
+    LAST=$(aws s3 ls s3://${BUCKET}/backup_redis/ | tail -1 | awk '{ print $4 }')
+    aws s3 rm s3://${BUCKET}/backup_redis/${LAST}
+    NUM_REDIS=$((NUM_REDIS - 1))
+done
+
+while [[ $NUM_POSTGRES -gt $MAX_DAILY ]]; do
+    LAST=$(aws s3 ls s3://${BUCKET}/backup_postgres/ | tail -1 | awk '{ print $4 }')
+    aws s3 rm s3://${BUCKET}/backup_postgres/${LAST}
+    NUM_POSTGRES=$((NUM_POSTGRES - 1))
+done
+

terraform/outputs.tf

@@ -12,6 +12,9 @@ output "nameservers" {
 output "s3_bucket_name" {
   value = aws_s3_bucket.s3_bucket.id
 }
+output "s3_private_bucket_name" {
+  value = module.private_s3_bucket.s3_bucket_id
+}
 output "my_ip" {
   value = "${chomp(data.http.myip.response_body)}"
 }