From 8797b9f76e1ea06a6d3b7d7ee910c2cbe71bc0b2 Mon Sep 17 00:00:00 2001 From: Kuba Orlik Date: Fri, 12 Sep 2025 21:00:45 +0200 Subject: [PATCH] Initial commit --- meta/main.yml | 17 +++ tasks/main.yml | 159 ++++++++++++++++++++ templates/backup-is-restore-needed.sh.j2 | 13 ++ templates/backup-mount.sh.j2 | 13 ++ templates/backup-restore-if-necessary.sh.j2 | 7 + templates/backup-restore.sh.j2 | 57 +++++++ templates/backup-run.sh.j2 | 11 ++ templates/backup-send.sh.j2 | 17 +++ templates/backup-vars.sh.j2 | 5 + templates/rclone.conf.j2 | 9 ++ 10 files changed, 308 insertions(+) create mode 100644 meta/main.yml create mode 100644 tasks/main.yml create mode 100644 templates/backup-is-restore-needed.sh.j2 create mode 100644 templates/backup-mount.sh.j2 create mode 100644 templates/backup-restore-if-necessary.sh.j2 create mode 100644 templates/backup-restore.sh.j2 create mode 100644 templates/backup-run.sh.j2 create mode 100644 templates/backup-send.sh.j2 create mode 100644 templates/backup-vars.sh.j2 create mode 100644 templates/rclone.conf.j2 diff --git a/meta/main.yml b/meta/main.yml new file mode 100644 index 0000000..50a843e --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,17 @@ +--- +galaxy_info: + author: kuba + description: sets up backups using restic to an S3 instance + license: MIT + min_ansible_version: "2.12" + platforms: + - name: Debian + versions: + - bullseye + - bookworm + - name: Ubuntu + versions: + - focal + - jammy + +dependencies: [] diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..d9ad759 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,159 @@ +- debug: + var: group_names + +- set_fact: + all_backup_paths: "{{ all_backup_paths | default([]) + (lookup('file', 'inventory/group_vars/' + item + '.yml') | from_yaml | dict2items | selectattr('key', 'equalto', 'backup_paths') | map(attribute='value') | list | first | default([])) }}" + loop: "{{ group_names }}" + +- debug: + var: all_backup_paths + +- name: make sure restic is installed + apt: state=latest pkg=restic + +- name: make sure fuse is installed (for mounting backups) + apt: state=latest pkg=fuse + +- name: save backup password + copy: + dest: "/backup-pwd" + content: "{{ BACKUP_PASSWORD }}" + mode: "0400" + +- name: Install boto3 and botocore using apt + become: yes + apt: + name: + - rclone + state: present + update_cache: yes + +- name: Install boto3 and botocore using apt + become: yes + apt: + name: + - python3-boto3 + - python3-botocore + state: present + +- name: Ensure rclone directory exists + file: + path: "/root/.config/rclone/" + state: directory + recurse: yes + mode: "0700" + +- name: Create rclone config + template: + src: "rclone.conf.j2" + dest: /root/.config/rclone/rclone.conf + mode: "0400" + force: yes + backup: yes + +- name: "Create a bucket for the backups" + amazon.aws.s3_bucket: + name: "icd-backup-{{ inventory_hostname }}" + state: present + endpoint_url: "{{ cloudflare_r2_endpoint }}" + access_key: "{{ cloudflare_r2_access_key }}" + secret_key: "{{ cloudflare_r2_secret_key }}" + +- name: initiate restic repository + command: "restic init --password-file=/backup-pwd" + register: command_result + retries: 3 + delay: 3 + until: "command_result.rc==0 or 'repository master key and config already initialized' in command_result.stderr" + failed_when: "command_result.rc!=0 and 'repository master key and config already initialized' not in command_result.stderr" + changed_when: "false" + environment: + RESTIC_REPOSITORY: "s3:{{ cloudflare_r2_endpoint }}/icd-backup-{{ inventory_hostname }}" + AWS_ACCESS_KEY_ID: "{{ cloudflare_r2_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ cloudflare_r2_secret_key }}" + tags: + - initiate + - connection_sanity + +- name: Create the backup vars script + template: + src: "backup-vars.sh.j2" + dest: /root/backup-vars.sh + mode: "0400" + force: yes + backup: yes + +- name: Create the backup send script + template: + src: "backup-send.sh.j2" + dest: /root/backup-send.sh + mode: u+rwx + force: yes + backup: yes + +- name: Create the backup mount script + template: + src: "backup-mount.sh.j2" + dest: /root/backup-mount.sh + mode: u+rwx + force: yes + backup: yes + tags: + - mount-script + +- name: Create the backup restore script + template: + src: "backup-restore.sh.j2" + dest: /root/backup-restore.sh + mode: u+rwx + force: yes + backup: yes + +- name: Create the backup prepare script + ansible.builtin.template: + src: "backup-scripts/{{inventory_hostname}}.sh.j2" + dest: /root/backup-prepare.sh + mode: u+rwx + backup: yes + force: yes + +- name: Create is-restore-needed script + ansible.builtin.template: + src: "backup-is-restore-needed.sh.j2" + dest: /root/backup-is-restore-needed.sh + mode: u+rwx + backup: yes + force: yes + +- name: Create the backup run script + ansible.builtin.template: + src: "backup-run.sh.j2" + dest: /root/backup-run.sh + mode: u+rwx + backup: yes + force: yes + +- name: Create the restore-if-needed script + ansible.builtin.template: + src: "backup-restore-if-necessary.sh.j2" + dest: /root/backup-restore-if-necessary.sh + mode: u+rwx + backup: yes + force: yes + +- name: setup CRON + ansible.builtin.cron: + name: "nightly backup for {{ inventory_hostname }}" + minute: 15 + hour: 4 + job: "/root/backup-run.sh" + +- name: "Restore backup if necessary" + command: /root/backup-restore-if-necessary.sh + register: command_output + args: + chdir: /root + +- name: "Print command output" + debug: + var: command_output diff --git a/templates/backup-is-restore-needed.sh.j2 b/templates/backup-is-restore-needed.sh.j2 new file mode 100644 index 0000000..a4def21 --- /dev/null +++ b/templates/backup-is-restore-needed.sh.j2 @@ -0,0 +1,13 @@ +#!/bin/bash + +# returns code 0 if backup is necessary, 1 otherwise + +eval "$DIRS_TO_BACKUP_STR" + +for file in "${DIRS_TO_BACKUP[@]}"; do + if [ ! -e "$file" ]; then + exit 0 + fi +done + +exit 1 diff --git a/templates/backup-mount.sh.j2 b/templates/backup-mount.sh.j2 new file mode 100644 index 0000000..b03ac7b --- /dev/null +++ b/templates/backup-mount.sh.j2 @@ -0,0 +1,13 @@ +source ./backup-vars.sh + +MOUNT_PATH="/mnt/restic" + +mkdir -p "$MOUNT_PATH" + + +###### Unlock the restic database in case it's locked + +$RESTIC --password-file=$PWD_FILE unlock + +$RESTIC --password-file=$PWD_FILE mount "$MOUNT_PATH" + diff --git a/templates/backup-restore-if-necessary.sh.j2 b/templates/backup-restore-if-necessary.sh.j2 new file mode 100644 index 0000000..2887f43 --- /dev/null +++ b/templates/backup-restore-if-necessary.sh.j2 @@ -0,0 +1,7 @@ +#!/bin/bash + +source ./backup-vars.sh; + +if ./backup-is-restore-needed.sh; then + ./backup-restore.sh +fi diff --git a/templates/backup-restore.sh.j2 b/templates/backup-restore.sh.j2 new file mode 100644 index 0000000..c4da9ed --- /dev/null +++ b/templates/backup-restore.sh.j2 @@ -0,0 +1,57 @@ +#!/bin/bash + +original_dir=$(pwd) + +source ./backup-vars.sh +cd "$original_dir" + +./backup-mount.sh & +RESTIC_PID=$! + +MOUNT_DIR="/mnt/restic" + +# Wait until the directory exists +while [ ! -d "$MOUNT_DIR" ]; do + echo "Waiting for directory to be created..." + sleep 1 +done + +# Wait until the directory is not empty +while [ ! "$(ls -A "$MOUNT_DIR")" ]; do + echo "Waiting for directory to have content..." + sleep 1 +done + +# Space-separated list of absolute paths +BACKUP_DIR="$MOUNT_DIR/snapshots/latest" + +eval "$DIRS_TO_BACKUP_STR" # creates the DIRS_TO_BACKUP array from string + +# Iterate over each path in the array +for ORIGINAL_PATH in "${DIRS_TO_BACKUP[@]}"; do + # Skip empty paths + [[ -z "$ORIGINAL_PATH" ]] && continue + + # Construct the corresponding backup path + BACKUP_PATH="$BACKUP_DIR$ORIGINAL_PATH" + + # Check if the backup directory exists + if [[ ! -d "$BACKUP_PATH" ]]; then + echo "Backup directory not found: $BACKUP_PATH" + continue + fi + + # Ensure the original directory exists, create it if not + if [[ ! -d "$ORIGINAL_PATH" ]]; then + echo "Creating original directory: $ORIGINAL_PATH" + mkdir -p "$ORIGINAL_PATH" + fi + + # Use rsync to copy files, preserving permissions, ownership, and timestamps + rsync --archive --acls --xattrs --compress --verbose --human-readable --partial --progress "$BACKUP_PATH/" "$ORIGINAL_PATH/" +done + +kill -SIGINT $RESTIC_PID +umount /mnt/restic + +echo "DONE" diff --git a/templates/backup-run.sh.j2 b/templates/backup-run.sh.j2 new file mode 100644 index 0000000..1259b22 --- /dev/null +++ b/templates/backup-run.sh.j2 @@ -0,0 +1,11 @@ +#!/bin/bash + +original_dir=$(pwd) + +source ./backup-vars.sh +cd "$original_dir" +date +source ./backup-prepare.sh +cd "$original_dir" +date +source ./backup-send.sh diff --git a/templates/backup-send.sh.j2 b/templates/backup-send.sh.j2 new file mode 100644 index 0000000..8163026 --- /dev/null +++ b/templates/backup-send.sh.j2 @@ -0,0 +1,17 @@ +###### Unlock the restic database in case it's locked + +$RESTIC --password-file=$PWD_FILE unlock + +###### Send backups + +date +echo "Sending the backup to the destination..." + +eval "$DIRS_TO_BACKUP_STR" # turn the string into an array + +$RESTIC --password-file=$PWD_FILE backup "${DIRS_TO_BACKUP[@]}" + +date +echo "Pruning the backup on the destination..." + +$RESTIC --password-file=$PWD_FILE forget --prune --keep-daily 3 --keep-weekly 2 --keep-monthly 1 --keep-yearly 1 diff --git a/templates/backup-vars.sh.j2 b/templates/backup-vars.sh.j2 new file mode 100644 index 0000000..2eebe4d --- /dev/null +++ b/templates/backup-vars.sh.j2 @@ -0,0 +1,5 @@ +declare -a DIRS_TO_BACKUP=({% for item in all_backup_paths %}"{{ item }}"{% if not loop.last %} {% endif %}{% endfor %}) +export DIRS_TO_BACKUP_STR=$(declare -p DIRS_TO_BACKUP) +export RESTIC_REPOSITORY="rclone:cloudflare-r2:icd-backup-{{ inventory_hostname }}" +export PWD_FILE=/backup-pwd +export RESTIC=/usr/bin/restic diff --git a/templates/rclone.conf.j2 b/templates/rclone.conf.j2 new file mode 100644 index 0000000..4ba9003 --- /dev/null +++ b/templates/rclone.conf.j2 @@ -0,0 +1,9 @@ +[cloudflare-r2] +type = s3 +provider = Cloudflare +access_key_id = {{ cloudflare_r2_access_key }} +secret_access_key = {{ cloudflare_r2_secret_key }} +region = auto +endpoint = {{ cloudflare_r2_endpoint }} +bucket_acl = private +