Differences

This shows you the differences between two versions of the page.

--- postgresql_barman_configuration [2025/03/20 19:10] – andonovj
+++ postgresql_barman_configuration [2025/05/11 18:12] (current) – [Backup to OCI] andonovj
@@ Line 554: / Line 554: @@
 </sxh>
+=====Backup to OCI=====
+I had a task to backup using barman, encrypt, upload and house keep the backups.
+So I implemented a crontab, that do just that:
+<Code:bash | Backup script>
+#!/bin/bash
+switch_logfiles(){
+    v_leader=$1
+    while :
+    do
+        echo "Switch logfile"
+        psql -Ubarman -h ${v_leader} -c "select pg_switch_wal()" -d datamine
+        sleep 10
+    done
+}
+backup() {
+environment=$1
+echo "Backup start"
+echo "Obtaining the first replica in the cluster"
+case ${environment} in
+    "dev")
+    ;;
+    "stg")
+    ;;
+    "prod")
+    ;;
+    *)
+        echo "Environment not given"
+    ;;
+esac
+v_first_replica=$(curl -sk https://${REPLICA_HOST}:8008/cluster | jq -r '.members[] | select(.role == "replica") | .host' | head -n 1)
+v_leader=$(curl -sk https://${REPLICA_HOST}:8008/cluster | jq -r '.members[] | select(.role == "leader") | .host')
+echo "Replacing placeholder"
+sed -i "s/REPLICA_IP_PLACEHOLDER/${v_first_replica}/g" /etc/barman.d/${environment}/streaming-server.conf
+echo "Creating replication slot"
+barman -c /etc/barman_${environment}.conf receive-wal --create-slot ${environment}
+echo "Starting cron and receive-wal"
+barman -c /etc/barman_${environment}.conf cron
+echo "Backup process start"
+switch_logfiles ${v_leader} &
+barman -c /etc/barman_${environment}.conf backup ${environment} --wait
+kill $!
+echo "Backup Ends"
+echo "Return placeholder"
+sed -i "s/${v_first_replica}/REPLICA_IP_PLACEHOLDER/g" /etc/barman.d/${environment}/streaming-server.conf
+}
+encrypt_backup(){
+        echo "Encryption start"
+        date '+%Y-%m-%d %H:%M:%S'
+        environment=$1
+        backup_dir=$(ls -rtd /data/backup/${environment}/base/*/* | tail -1)
+        cd $(dirname ${backup_dir})
+        backup_file=$(echo ${backup_dir} | cut -d '/' -f 6)
+        enc_key=$(cat /var/lib/barman/enckey.enc | base64 -d)
+        tar cvf - data/ | openssl enc -e -pbkdf2 -aes-256-ctr -k ${enc_key} -out ${backup_file}.tar.enc
+        echo "Encryption Ends"
+        date '+%Y-%m-%d %H:%M:%S'
+}
+cleanup() {
+    environment=$1
+    BUCKET_NAME=postgres-${environment}-backup
+    OCI_CONFIG_FILE=/var/lib/barman/ocivault/#{environment}.conf
+    DATE_90_DAYS_AGO=$(date -d "90 days ago" +"%Y-%m-%d")
+    for object_name in $(oci os object list --bucket-name ${BUCKET_NAME} | jq -r --arg DATE_90_DAYS_AGO "$DATE_90_DAYS_AGO" '.data[] | select ( (."time-created"[:10] | strptime("%Y-%m-%d") | mktime) < ($DATE_90_DAYS_AGO | strptime("%Y-%m-%d") | mktime )) | .name');
+    do
+        oci os object delete --bucket-name ${BUCKET_NAME} --object-name ${object_name} --force
+    done
+    DATE_35_DAYS_AGO=$(date -d "35 days ago" +"%Y-%m-%d")
+    for object_name in $(oci os object list --bucket-name ${BUCKET_NAME} | jq -r --arg DATE_35_DAYS_AGO "$DATE_35_DAYS_AGO" '.data[] | select ( ((."time-created"[:10] | strptime("%Y-%m-%d") | mktime) > ($DATE_35_DAYS_AGO | strptime("%Y-%m-%d") | mktime)) and (."time-created"[:10] | strptime("%Y-%m-%d") | mktime | strftime("%u") | tonumber % 7 != 0) )');
+    do
+        oci os object delete --bucket-name ${BUCKET_NAME} --object-name ${object_name} --force
+    done
+}
+copy_files_to_os(){
+        echo "Copy to Bucket starts"
+        date '+%Y-%m-%d %H:%M:%S'
+        environment=$1
+        echo "Set BUCKET_NAME and OCI config file variable"
+        case ${environment} in
+            "dev")
+                BUCKET_NAME=postgres-dev-backup
+                OCI_CONFIG_FILE=/var/lib/barman/ocivault/dev.conf
+            ;;
+            "stg")
+                BUCKET_NAME=postgres-stg-backup
+                OCI_CONFIG_FILE=/var/lib/barman/ocivault/stg.conf
+            ;;
+            "prod")
+                BUCKET_NAME=postgres-prod-backup
+                OCI_CONFIG_FILE=/var/lib/barman/ocivault/prod.conf
+            ;;
+            *)
+        esac
+        export BUCKET_NAME
+        export OCI_CONFIG_FILE
+        encrypted_file=$(ls -rtd /data/backup/${environment}/base/*/* | grep "tar.enc" | tail -1)
+        oci os object put --bucket-name ${BUCKET_NAME} --file ${encrypted_file}
+        rm -f ${encrypted_file}
+        echo "Copy to Bucket ends"
+        date '+%Y-%m-%d %H:%M:%S'
+}
+backup $1
+encrypt_backup $1
+copy_files_to_os $1
+cleanup $1
+</Code>
+The script is used very simple:
+bash script.sh "dev"