From 7a7b5a723a3b2a431fb1f5af6a4a354bcbaba53d Mon Sep 17 00:00:00 2001 From: Geraldo Date: Thu, 2 Apr 2026 09:04:01 -0300 Subject: [PATCH] Backup and restore module, implemented on optscale --- .../backup_module/backup/Dockerfile | 64 +++++ .../backup_module/backup/backup.sh | 219 ++++++++++++++++++ .../backup_module/backup/cronjob.yaml | 69 ++++++ optscale-deploy/backup_module/backup/job.yaml | 65 ++++++ .../backup_module/restore/Dockerfile | 26 +++ .../backup_module/restore/restore-pod.yaml | 54 +++++ .../backup_module/restore/restore.sh | 210 +++++++++++++++++ optscale-deploy/backup_module/secrets.yaml | 14 ++ 8 files changed, 721 insertions(+) create mode 100644 optscale-deploy/backup_module/backup/Dockerfile create mode 100644 optscale-deploy/backup_module/backup/backup.sh create mode 100644 optscale-deploy/backup_module/backup/cronjob.yaml create mode 100644 optscale-deploy/backup_module/backup/job.yaml create mode 100644 optscale-deploy/backup_module/restore/Dockerfile create mode 100644 optscale-deploy/backup_module/restore/restore-pod.yaml create mode 100644 optscale-deploy/backup_module/restore/restore.sh create mode 100644 optscale-deploy/backup_module/secrets.yaml diff --git a/optscale-deploy/backup_module/backup/Dockerfile b/optscale-deploy/backup_module/backup/Dockerfile new file mode 100644 index 000000000..3649c3594 --- /dev/null +++ b/optscale-deploy/backup_module/backup/Dockerfile @@ -0,0 +1,64 @@ +# Stage 1: Builder +FROM debian:bookworm-slim AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + unzip \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install AWS CLI v2 (installs to /usr/local/aws-cli by default) +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + ./aws/install && \ + rm -rf awscliv2.zip aws + +# Download and extract MongoDB tools +RUN curl -LO https://fastdl.mongodb.org/tools/db/mongodb-database-tools-debian12-x86_64-100.10.0.tgz && \ + tar -xzf mongodb-database-tools-debian12-x86_64-100.10.0.tgz && \ + mv mongodb-database-tools-*/bin/* /usr/local/bin/ && \ + rm -rf mongodb-database-tools-* + +# Install ClickHouse client (static binary) +RUN curl -LO https://github.com/ClickHouse/ClickHouse/releases/latest/download/clickhouse-client && \ + chmod +x clickhouse-client && \ + mv clickhouse-client /usr/local/bin/clickhouse-client + + +# Stage 2: Final lightweight image +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + default-mysql-client \ + redis-tools \ + curl \ + ca-certificates \ + bash \ + groff \ + less \ + libgssapi-krb5-2 \ + && rm -rf /var/lib/apt/lists/* + +# Copy AWS CLI v2 (full install dir + symlink) +COPY --from=builder /usr/local/aws-cli /usr/local/aws-cli +RUN ln -s /usr/local/aws-cli/v2/current/bin/aws /usr/local/bin/aws + +# Copy MongoDB tools +COPY --from=builder /usr/local/bin/mongodump \ + /usr/local/bin/mongorestore \ + /usr/local/bin/mongoexport \ + /usr/local/bin/mongoimport \ + /usr/local/bin/mongostat \ + /usr/local/bin/mongotop \ + /usr/local/bin/bsondump \ + /usr/local/bin/mongofiles \ + /usr/local/bin/ + +# Copy ClickHouse client +COPY --from=builder /usr/local/bin/clickhouse-client /usr/local/bin/clickhouse-client + +# Copy backup script +COPY backup.sh /usr/local/bin/backup.sh +RUN chmod +x /usr/local/bin/backup.sh + +ENTRYPOINT ["/usr/local/bin/backup.sh"] diff --git a/optscale-deploy/backup_module/backup/backup.sh b/optscale-deploy/backup_module/backup/backup.sh new file mode 100644 index 000000000..d0c12cb07 --- /dev/null +++ b/optscale-deploy/backup_module/backup/backup.sh @@ -0,0 +1,219 @@ +#!/bin/bash +set -uo pipefail + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +BACKUP_DIR="/tmp/backups/$TIMESTAMP" +CLICKHOUSE_DATA_DIR="$BACKUP_DIR/clickhouse_data" +mkdir -p "$BACKUP_DIR" "$CLICKHOUSE_DATA_DIR" + +# Track what succeeded/failed for summary +MARIADB_STATUS="skipped" +MONGO_STATUS="skipped" +CLICKHOUSE_STATUS="skipped" +RABBITMQ_STATUS="skipped" +UPLOAD_STATUS="skipped" + +echo "--- Starting Full Database Dump $TIMESTAMP ---" + +# Required env vars: +# MARIADB_ROOT_PASSWORD, MONGO_ROOT_PASSWORD, CLICKHOUSE_PASSWORD, RABBITMQ_PASSWORD +# S3_BUCKET, S3_PREFIX +# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION + +# ───────────────────────────────────────────── +# 1. MariaDB (schema + data) +# ───────────────────────────────────────────── +echo "" +echo "[1/5] Backing up MariaDB..." +if mysqldump \ + -h mariadb \ + -u root \ + -p"$MARIADB_ROOT_PASSWORD" \ + --all-databases \ + --single-transaction \ + --routines \ + --triggers \ + --events \ + > "$BACKUP_DIR/mariadb.sql" 2>/tmp/mariadb_err.txt; then + echo "MariaDB backup complete." + MARIADB_STATUS="ok" +else + echo "ERROR: MariaDB backup failed. Skipping." + cat /tmp/mariadb_err.txt || true + rm -f "$BACKUP_DIR/mariadb.sql" + MARIADB_STATUS="FAILED" +fi + +# ───────────────────────────────────────────── +# 2. MongoDB (data only, no auth/user metadata) +# ───────────────────────────────────────────── +echo "" +echo "[2/5] Backing up MongoDB..." +if mongodump \ + --host mongo-discovery \ + --username root \ + --password "$MONGO_ROOT_PASSWORD" \ + --authenticationDatabase admin \ + --db restapi \ + --archive="$BACKUP_DIR/mongo.archive" \ + 2>/tmp/mongo_err.txt; then + echo "MongoDB backup complete." + MONGO_STATUS="ok" +else + echo "ERROR: MongoDB backup failed. Skipping." + cat /tmp/mongo_err.txt || true + rm -f "$BACKUP_DIR/mongo.archive" + MONGO_STATUS="FAILED" +fi + +# ───────────────────────────────────────────── +# 3. ClickHouse (schemas + table data) +# ───────────────────────────────────────────── +echo "" +echo "[3/5] Backing up ClickHouse schemas and data..." + +CH_FAIL=0 + +mapfile -t CLICKHOUSE_DATABASES < <( + clickhouse-client \ + --host clickhouse \ + --user default \ + --password "$CLICKHOUSE_PASSWORD" \ + --query "SHOW DATABASES" 2>/tmp/ch_err.txt \ + | grep -Ev '^(system|information_schema|INFORMATION_SCHEMA)$' +) || CH_FAIL=1 + +if [ "$CH_FAIL" -eq 1 ]; then + echo "ERROR: Could not connect to ClickHouse. Skipping." + cat /tmp/ch_err.txt || true + CLICKHOUSE_STATUS="FAILED" +else + CLICKHOUSE_STATUS="ok" + for db in "${CLICKHOUSE_DATABASES[@]}"; do + [ -z "$db" ] && continue + + echo " Backing up ClickHouse database schema: $db" + if ! clickhouse-client \ + --host clickhouse \ + --user default \ + --password "$CLICKHOUSE_PASSWORD" \ + --query "SHOW CREATE DATABASE \`$db\`" \ + > "$BACKUP_DIR/clickhouse_${db}_schema.sql" 2>/tmp/ch_err.txt; then + echo " WARNING: Failed to back up schema for database $db. Skipping database." + cat /tmp/ch_err.txt || true + CLICKHOUSE_STATUS="partial" + continue + fi + + DB_DIR="$CLICKHOUSE_DATA_DIR/$db" + mkdir -p "$DB_DIR" + + mapfile -t TABLES < <( + clickhouse-client \ + --host clickhouse \ + --user default \ + --password "$CLICKHOUSE_PASSWORD" \ + --query "SHOW TABLES FROM \`$db\`" 2>/tmp/ch_err.txt + ) || { echo " WARNING: Could not list tables for $db. Skipping."; cat /tmp/ch_err.txt || true; continue; } + + for table in "${TABLES[@]}"; do + [ -z "$table" ] && continue + + echo " Backing up ClickHouse table schema: $db.$table" + if ! clickhouse-client \ + --host clickhouse \ + --user default \ + --password "$CLICKHOUSE_PASSWORD" \ + --query "SHOW CREATE TABLE \`$db\`.\`$table\`" \ + > "$DB_DIR/${table}.schema.sql" 2>/tmp/ch_err.txt; then + echo " WARNING: Failed schema for $db.$table. Skipping table." + cat /tmp/ch_err.txt || true + CLICKHOUSE_STATUS="partial" + continue + fi + + echo " Backing up ClickHouse table data: $db.$table" + if ! clickhouse-client \ + --host clickhouse \ + --user default \ + --password "$CLICKHOUSE_PASSWORD" \ + --query "SELECT * FROM \`$db\`.\`$table\` FORMAT Native" \ + > "$DB_DIR/${table}.native" 2>/tmp/ch_err.txt; then + echo " WARNING: Failed data dump for $db.$table. Skipping table data." + cat /tmp/ch_err.txt || true + rm -f "$DB_DIR/${table}.native" + CLICKHOUSE_STATUS="partial" + fi + done + done + + echo "ClickHouse backup complete (status: $CLICKHOUSE_STATUS)." +fi + +# ───────────────────────────────────────────── +# 4. RabbitMQ (definitions only) +# ───────────────────────────────────────────── +echo "" +echo "[4/5] Backing up RabbitMQ definitions..." + +HTTP_STATUS=$(curl -s -o "$BACKUP_DIR/rabbitmq_defs.json" -w "%{http_code}" \ + -u optscale:"$RABBITMQ_PASSWORD" \ + http://rabbitmq:15672/api/definitions 2>/tmp/rabbitmq_err.txt) || HTTP_STATUS="000" + +if [ "$HTTP_STATUS" = "200" ]; then + echo "RabbitMQ definitions backup complete." + echo "NOTE: Queue message contents are not included." + RABBITMQ_STATUS="ok" +else + echo "ERROR: RabbitMQ backup failed with HTTP $HTTP_STATUS. Skipping." + cat /tmp/rabbitmq_err.txt || true + rm -f "$BACKUP_DIR/rabbitmq_defs.json" + RABBITMQ_STATUS="FAILED (HTTP $HTTP_STATUS)" +fi + +# ───────────────────────────────────────────── +# 5. Manifest + Upload to S3 +# ───────────────────────────────────────────── +echo "" +echo "[5/5] Creating manifest and uploading to S3..." + +{ + echo "timestamp=$TIMESTAMP" + echo "created_at=$(date -Iseconds)" + echo "mariadb=$MARIADB_STATUS" + echo "mongodb=$MONGO_STATUS" + echo "clickhouse=$CLICKHOUSE_STATUS" + echo "rabbitmq=$RABBITMQ_STATUS" + echo "---files---" + find "$BACKUP_DIR" -maxdepth 5 -type f | sort +} > "$BACKUP_DIR/backup_manifest.txt" + +if aws s3 cp "$BACKUP_DIR/" "s3://$S3_BUCKET/$S3_PREFIX/$TIMESTAMP/" --recursive 2>/tmp/s3_err.txt; then + echo "Backup uploaded to s3://$S3_BUCKET/$S3_PREFIX/$TIMESTAMP/" + UPLOAD_STATUS="ok" +else + echo "ERROR: S3 upload failed." + cat /tmp/s3_err.txt || true + UPLOAD_STATUS="FAILED" +fi + +# ───────────────────────────────────────────── +# Summary +# ───────────────────────────────────────────── +echo "" +echo "======================================" +echo " Backup Summary — $TIMESTAMP" +echo "======================================" +echo " MariaDB : $MARIADB_STATUS" +echo " MongoDB : $MONGO_STATUS" +echo " ClickHouse : $CLICKHOUSE_STATUS" +echo " RabbitMQ : $RABBITMQ_STATUS" +echo " S3 Upload : $UPLOAD_STATUS" +echo "======================================" + +if [ "$UPLOAD_STATUS" = "FAILED" ]; then + echo "CRITICAL: S3 upload failed. Exiting with error." + exit 1 +fi + +echo "Backup job finished." diff --git a/optscale-deploy/backup_module/backup/cronjob.yaml b/optscale-deploy/backup_module/backup/cronjob.yaml new file mode 100644 index 000000000..9bfc9c234 --- /dev/null +++ b/optscale-deploy/backup_module/backup/cronjob.yaml @@ -0,0 +1,69 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: all-db-backup + namespace: default +spec: + schedule: "0 2 * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + ttlSecondsAfterFinished: 60 + template: + spec: + restartPolicy: OnFailure + containers: + - name: backup-worker + image: registry-git.lsd.ufcg.edu.br/vtex-lab/mokirana/backup:latest + env: + - name: S3_BUCKET + value: "mokirana-backups" + - name: S3_PREFIX + value: "full-infrastructure" + - name: AWS_DEFAULT_REGION + value: "us-east-1" + - name: MARIADB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: MARIADB_ROOT_PASSWORD + - name: MONGO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: MONGO_ROOT_PASSWORD + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: CLICKHOUSE_PASSWORD + - name: RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: RABBITMQ_PASSWORD + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_SECRET_ACCESS_KEY + - name: AWS_SESSION_TOKEN + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_SESSION_TOKEN + volumes: + - name: backup-storage + ephemeral: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "gp3-tagged" + resources: + requests: + storage: 1Ti diff --git a/optscale-deploy/backup_module/backup/job.yaml b/optscale-deploy/backup_module/backup/job.yaml new file mode 100644 index 000000000..4c4b1f567 --- /dev/null +++ b/optscale-deploy/backup_module/backup/job.yaml @@ -0,0 +1,65 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: all-db-backup-manual + namespace: default +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + restartPolicy: OnFailure + containers: + - name: backup-worker + image: registry-git.lsd.ufcg.edu.br/vtex-lab/mokirana/backup:latest + env: + - name: S3_BUCKET + value: "mokirana-backups" + - name: S3_PREFIX + value: "full-infrastructure" + - name: AWS_DEFAULT_REGION + value: "us-east-1" + - name: MARIADB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: MARIADB_ROOT_PASSWORD + - name: MONGO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: MONGO_ROOT_PASSWORD + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: CLICKHOUSE_PASSWORD + - name: RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: RABBITMQ_PASSWORD + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_SECRET_ACCESS_KEY + - name: AWS_SESSION_TOKEN + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_SESSION_TOKEN + volumes: + - name: backup-storage + ephemeral: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: "gp3-tagged" + resources: + requests: + storage: 1Ti diff --git a/optscale-deploy/backup_module/restore/Dockerfile b/optscale-deploy/backup_module/restore/Dockerfile new file mode 100644 index 000000000..f751c5c46 --- /dev/null +++ b/optscale-deploy/backup_module/restore/Dockerfile @@ -0,0 +1,26 @@ +FROM alpine:3.19 AS builder + +RUN apk add --no-cache curl bash + +# Download ClickHouse static binary +RUN curl https://clickhouse.com/ | sh \ + && mv clickhouse /usr/local/bin/clickhouse \ + && ln -s /usr/local/bin/clickhouse /usr/local/bin/clickhouse-client + +FROM alpine:3.19 + +RUN apk add --no-cache \ + bash \ + curl \ + mariadb-client \ + mongodb-tools \ + aws-cli \ + ca-certificates + +COPY --from=builder /usr/local/bin/clickhouse /usr/local/bin/clickhouse +COPY --from=builder /usr/local/bin/clickhouse-client /usr/local/bin/clickhouse-client + +COPY restore.sh /usr/local/bin/restore.sh +RUN chmod +x /usr/local/bin/restore.sh + +ENTRYPOINT ["/usr/local/bin/restore.sh"] diff --git a/optscale-deploy/backup_module/restore/restore-pod.yaml b/optscale-deploy/backup_module/restore/restore-pod.yaml new file mode 100644 index 000000000..bce47264c --- /dev/null +++ b/optscale-deploy/backup_module/restore/restore-pod.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: Pod +metadata: + name: db-restore + namespace: default +spec: + restartPolicy: Never + containers: + - name: restore + image: registry-git.lsd.ufcg.edu.br/vtex-lab/mokirana/restore:latest + env: + - name: S3_BUCKET + value: "mokirana-backups" + - name: S3_PREFIX + value: "full-infrastructure" + - name: TIMESTAMP + value: "20260320_142016" # Set to the backup timestamp you want to restore + - name: AWS_DEFAULT_REGION + value: "us-east-1" + - name: MARIADB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: MARIADB_ROOT_PASSWORD + - name: MONGO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: MONGO_ROOT_PASSWORD + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: CLICKHOUSE_PASSWORD + - name: RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: RABBITMQ_PASSWORD + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_SECRET_ACCESS_KEY + - name: AWS_SESSION_TOKEN + valueFrom: + secretKeyRef: + name: multi-db-backup-secrets + key: AWS_SESSION_TOKEN diff --git a/optscale-deploy/backup_module/restore/restore.sh b/optscale-deploy/backup_module/restore/restore.sh new file mode 100644 index 000000000..ada10ee71 --- /dev/null +++ b/optscale-deploy/backup_module/restore/restore.sh @@ -0,0 +1,210 @@ +#!/bin/bash +set -euo pipefail + +# Required environment variables: +# S3_BUCKET, S3_PREFIX, TIMESTAMP +# MARIADB_ROOT_PASSWORD, MONGO_ROOT_PASSWORD, CLICKHOUSE_PASSWORD, RABBITMQ_PASSWORD +# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION +# Optional: AWS_SESSION_TOKEN + +BACKUP_DIR="/tmp/restore/$TIMESTAMP" +CLICKHOUSE_DATA_DIR="$BACKUP_DIR/clickhouse_data" +mkdir -p "$BACKUP_DIR" + +# Track what succeeded/failed for summary +MARIADB_STATUS="skipped" +MONGO_STATUS="skipped" +CLICKHOUSE_STATUS="skipped" +RABBITMQ_STATUS="skipped" + +echo "--- Starting Restore from S3 Backup $TIMESTAMP ---" + +# ───────────────────────────────────────────── +# Download backup files from S3 +# ───────────────────────────────────────────── +echo "" +echo "Downloading backup files from s3://$S3_BUCKET/$S3_PREFIX/$TIMESTAMP/" +aws s3 cp "s3://$S3_BUCKET/$S3_PREFIX/$TIMESTAMP/" "$BACKUP_DIR/" --recursive + +echo "Downloaded files:" +find "$BACKUP_DIR" -type f | sort + +# Validate manifest +if [ -f "$BACKUP_DIR/backup_manifest.txt" ]; then + echo "" + echo "Backup manifest:" + cat "$BACKUP_DIR/backup_manifest.txt" +else + echo "WARNING: No backup_manifest.txt found. Proceeding anyway." +fi + +echo "" + +# ───────────────────────────────────────────── +# 1. Restore MariaDB +# ───────────────────────────────────────────── +echo "[1/4] Restoring MariaDB..." +if [ -f "$BACKUP_DIR/mariadb.sql" ]; then + if mysql \ + -h mariadb \ + -u root \ + -p"$MARIADB_ROOT_PASSWORD" \ + < "$BACKUP_DIR/mariadb.sql" 2>/tmp/mariadb_err.txt; then + echo "MariaDB restore complete." + MARIADB_STATUS="ok" + else + echo "ERROR: MariaDB restore failed." + cat /tmp/mariadb_err.txt || true + MARIADB_STATUS="FAILED" + fi +else + echo "WARNING: mariadb.sql not found, skipping MariaDB restore." + MARIADB_STATUS="skipped (file missing)" +fi + +echo "" + +# ───────────────────────────────────────────── +# 2. Restore MongoDB (no auth/user metadata) +# ───────────────────────────────────────────── +echo "[2/4] Restoring MongoDB..." +if mongorestore \ + --host mongo-discovery \ + --username root \ + --password "$MONGO_ROOT_PASSWORD" \ + --authenticationDatabase admin \ + --db restapi \ + --archive="$BACKUP_DIR/mongo.archive" \ + --drop \ + 2>/tmp/mongo_err.txt; then + echo "MongoDB restore complete." + MONGO_STATUS="ok" + else + echo "ERROR: MongoDB restore failed." + cat /tmp/mongo_err.txt || true + MONGO_STATUS="FAILED" + fi +else + echo "WARNING: mongo.archive not found, skipping MongoDB restore." + MONGO_STATUS="skipped (file missing)" +fi + +echo "" + +# ───────────────────────────────────────────── +# 3. Restore ClickHouse (schemas + table data) +# ───────────────────────────────────────────── +echo "[3/4] Restoring ClickHouse..." +if [ ! -d "$CLICKHOUSE_DATA_DIR" ]; then + echo "WARNING: clickhouse_data directory not found, skipping ClickHouse restore." + CLICKHOUSE_STATUS="skipped (directory missing)" +else + CLICKHOUSE_STATUS="ok" + + # Restore database-level schemas first + for db_schema_file in "$BACKUP_DIR"/clickhouse_*_schema.sql; do + [ -f "$db_schema_file" ] || continue + echo " Restoring ClickHouse database schema: $(basename "$db_schema_file")" + if ! clickhouse-client \ + --host clickhouse \ + --user default \ + --password "$CLICKHOUSE_PASSWORD" \ + --multiquery < "$db_schema_file" 2>/tmp/ch_err.txt; then + echo " WARNING: Failed to restore schema $(basename "$db_schema_file")" + cat /tmp/ch_err.txt || true + CLICKHOUSE_STATUS="partial" + fi + done + + # Restore per-table schemas and data + for db_dir in "$CLICKHOUSE_DATA_DIR"/*/; do + [ -d "$db_dir" ] || continue + db=$(basename "$db_dir") + echo " Restoring ClickHouse tables for database: $db" + + # Restore table schemas first + for table_schema_file in "$db_dir"*.schema.sql; do + [ -f "$table_schema_file" ] || continue + table=$(basename "$table_schema_file" .schema.sql) + echo " Restoring table schema: $db.$table" + if ! clickhouse-client \ + --host clickhouse \ + --user default \ + --password "$CLICKHOUSE_PASSWORD" \ + --multiquery < "$table_schema_file" 2>/tmp/ch_err.txt; then + echo " WARNING: Failed schema for $db.$table" + cat /tmp/ch_err.txt || true + CLICKHOUSE_STATUS="partial" + fi + done + + # Restore table data + for table_data_file in "$db_dir"*.native; do + [ -f "$table_data_file" ] || continue + table=$(basename "$table_data_file" .native) + + if [ ! -s "$table_data_file" ]; then + echo " Skipping empty data file for: $db.$table" + continue + fi + + echo " Restoring table data: $db.$table" + if ! clickhouse-client \ + --host clickhouse \ + --user default \ + --password "$CLICKHOUSE_PASSWORD" \ + --query "INSERT INTO \`$db\`.\`$table\` FORMAT Native" \ + < "$table_data_file" 2>/tmp/ch_err.txt; then + echo " WARNING: Failed data restore for $db.$table" + cat /tmp/ch_err.txt || true + CLICKHOUSE_STATUS="partial" + fi + done + done + + echo "ClickHouse restore complete (status: $CLICKHOUSE_STATUS)." +fi + +echo "" + +# ───────────────────────────────────────────── +# 4. Restore RabbitMQ definitions +# ───────────────────────────────────────────── +echo "[4/4] Restoring RabbitMQ definitions..." +if [ -f "$BACKUP_DIR/rabbitmq_defs.json" ]; then + HTTP_STATUS=$(curl -s -o /tmp/rabbitmq_restore_response.txt -w "%{http_code}" \ + -u optscale:"$RABBITMQ_PASSWORD" \ + -H "Content-Type: application/json" \ + -X POST \ + -d @"$BACKUP_DIR/rabbitmq_defs.json" \ + http://rabbitmq:15672/api/definitions 2>/tmp/rabbitmq_err.txt) || HTTP_STATUS="000" + + if [ "$HTTP_STATUS" = "200" ] || [ "$HTTP_STATUS" = "201" ] || [ "$HTTP_STATUS" = "204" ]; then + echo "RabbitMQ definitions restored successfully (HTTP $HTTP_STATUS)." + RABBITMQ_STATUS="ok" + else + echo "ERROR: RabbitMQ restore failed with HTTP $HTTP_STATUS" + cat /tmp/rabbitmq_restore_response.txt || true + RABBITMQ_STATUS="FAILED (HTTP $HTTP_STATUS)" + fi +else + echo "WARNING: rabbitmq_defs.json not found, skipping RabbitMQ restore." + RABBITMQ_STATUS="skipped (file missing)" +fi + +echo "" + +# ───────────────────────────────────────────── +# Summary +# ───────────────────────────────────────────── +echo "======================================" +echo " Restore Summary — $TIMESTAMP" +echo "======================================" +echo " MariaDB : $MARIADB_STATUS" +echo " MongoDB : $MONGO_STATUS" +echo " ClickHouse : $CLICKHOUSE_STATUS" +echo " RabbitMQ : $RABBITMQ_STATUS" +echo "======================================" +echo "" +echo "NOTE: RabbitMQ queue message contents are not restorable via definitions export." +echo "--- Restore Completed ---" diff --git a/optscale-deploy/backup_module/secrets.yaml b/optscale-deploy/backup_module/secrets.yaml new file mode 100644 index 000000000..fabb87941 --- /dev/null +++ b/optscale-deploy/backup_module/secrets.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Secret +metadata: + name: multi-db-backup-secrets + namespace: default +type: Opaque +stringData: + MARIADB_ROOT_PASSWORD: "my-password-01" + MONGO_ROOT_PASSWORD: "SecurePassword-01-02" + CLICKHOUSE_PASSWORD: "secure-password-1-clk" + RABBITMQ_PASSWORD: "secure-password-here" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + AWS_SESSION_TOKEN: ""