Configuration Guide

Configuration Overview

Terraback uses a hierarchical configuration system that allows flexibility and control over all aspects of backup, recovery, and monitoring operations.

Configuration Hierarchy

1. Default Configuration (built-in)
2. Global Configuration (~/.terraback/config.yml)
3. Project Configuration (./.terraback/config.yml)
4. Environment Variables (TERRABACK_*)
5. Command-line Arguments

Higher levels override lower levels, giving you fine-grained control.

Configuration File Structure

Complete Configuration Example

# ~/.terraback/config.yml
version: 1
profile: production

# Core Settings
core:
  project_name: "my-infrastructure"
  terraform_version: "1.5.7"
  log_level: info  # debug, info, warn, error
  color_output: true
  confirm_destructive: true
  telemetry_enabled: true

# Storage Configuration
storage:
  # Backend type: s3, azure, gcp, local
  type: s3

  # S3 Configuration
  s3:
    bucket: "terraback-backups-prod"
    region: "us-east-1"
    prefix: "terraform-states/"
    storage_class: "INTELLIGENT_TIERING"
    server_side_encryption: "AES256"
    kms_key_id: "arn:aws:kms:us-east-1:123456789012:key/..."
    access_key: "${AWS_ACCESS_KEY_ID}"  # Can reference env vars
    secret_key: "${AWS_SECRET_ACCESS_KEY}"
    session_token: "${AWS_SESSION_TOKEN}"
    endpoint: ""  # For S3-compatible services
    force_path_style: false

  # Azure Configuration
  azure:
    account_name: "terrabackstorage"
    container_name: "backups"
    access_key: "${AZURE_STORAGE_ACCESS_KEY}"
    sas_token: ""
    connection_string: ""

  # GCP Configuration
  gcp:
    bucket: "terraback-backups"
    project: "my-project-id"
    credentials_path: "/path/to/credentials.json"
    storage_class: "STANDARD"
    location: "us-central1"

  # Local Configuration
  local:
    path: "~/.terraback/backups"
    compress: true
    encrypt: true

# Backup Settings
backup:
  # Automatic backup settings
  automatic:
    enabled: true
    interval: "1h"  # 5m, 1h, 6h, 1d, 1w
    on_apply: true  # Backup before terraform apply
    on_destroy: true  # Backup before terraform destroy

  # Retention policies
  retention:
    days: 30
    minimum_backups: 10
    maximum_backups: 100
    keep_hourly: 24
    keep_daily: 7
    keep_weekly: 4
    keep_monthly: 12

  # Compression settings
  compression:
    enabled: true
    algorithm: "gzip"  # gzip, bzip2, xz, zstd
    level: 6  # 1-9 (1=fastest, 9=best compression)

  # Encryption settings
  encryption:
    enabled: true
    algorithm: "AES256"  # AES256, AES192, AES128
    key_source: "managed"  # managed, custom
    custom_key_path: ""

  # Performance settings
  performance:
    parallel_uploads: 5
    chunk_size: "10MB"
    multipart_threshold: "100MB"
    max_bandwidth: "100MB/s"  # Bandwidth limiting

  # Naming and tagging
  naming:
    pattern: "backup_{timestamp}_{workspace}"
    timestamp_format: "%Y%m%d_%H%M%S"
    include_hostname: false
    include_git_hash: true

  tags:
    environment: "production"
    team: "infrastructure"
    managed_by: "terraback"
    cost_center: "engineering"

# Drift Detection
drift:
  enabled: true
  interval: "15m"

  # Detection settings
  detection:
    check_terraform_state: true
    check_actual_resources: true
    ignore_metadata_changes: true
    fail_on_drift: false

  # Resources to monitor/ignore
  resources:
    include:
      - "aws_instance.*"
      - "aws_s3_bucket.*"
      - "aws_rds_instance.*"
    exclude:
      - "aws_instance.temporary_*"
      - "random_*"

  # Alerting
  alerts:
    enabled: true
    channels:
      - type: email
        addresses:
          - "ops@example.com"
          - "security@example.com"
      - type: slack
        webhook_url: "${SLACK_WEBHOOK_URL}"
        channel: "#infrastructure-alerts"
      - type: pagerduty
        service_key: "${PAGERDUTY_SERVICE_KEY}"

  # Auto-remediation
  remediation:
    enabled: false
    auto_fix_safe: true  # Only fix non-destructive drift
    require_approval: true
    approval_timeout: "1h"

# Monitoring & Metrics
monitoring:
  # Metrics export
  metrics:
    enabled: true
    provider: "prometheus"  # prometheus, datadog, cloudwatch
    endpoint: "http://localhost:9090"
    interval: "30s"

  # Health checks
  health:
    enabled: true
    port: 8080
    path: "/health"

  # Audit logging
  audit:
    enabled: true
    path: "/var/log/terraback/audit.log"
    rotation: "daily"
    retention: "90d"
    format: "json"  # json, text

# Terraform Integration
terraform:
  # Binary configuration
  binary:
    path: "terraform"  # Path to terraform binary
    version_constraint: ">= 0.12"
    download_if_missing: true

  # Workspace settings
  workspace:
    default: "default"
    auto_select: true
    backup_all: false

  # Execution settings
  execution:
    parallelism: 10
    timeout: "30m"
    refresh_before_backup: true
    lock_timeout: "5m"

  # State management
  state:
    validate_before_backup: true
    check_remote_state: true
    pull_before_operations: true

# Network Settings
network:
  # Proxy configuration
  proxy:
    http_proxy: "${HTTP_PROXY}"
    https_proxy: "${HTTPS_PROXY}"
    no_proxy: "localhost,127.0.0.1"

  # Timeouts
  timeouts:
    connection: "30s"
    read: "5m"
    write: "5m"

  # Retry configuration
  retry:
    max_attempts: 3
    initial_delay: "1s"
    max_delay: "30s"
    multiplier: 2

# Security Settings
security:
  # Authentication
  auth:
    type: "api_key"  # api_key, oauth, saml
    api_key: "${TERRABACK_API_KEY}"
    oauth:
      provider: "github"
      client_id: ""
      client_secret: ""

  # Access control
  access_control:
    enabled: true
    default_permission: "read"
    rbac:
      - role: "admin"
        permissions: ["read", "write", "delete", "restore"]
      - role: "operator"
        permissions: ["read", "write", "restore"]
      - role: "viewer"
        permissions: ["read"]

  # Compliance
  compliance:
    require_mfa: true
    require_encryption: true
    audit_all_operations: true
    compliance_standards:
      - "SOC2"
      - "HIPAA"
      - "PCI-DSS"

# Notification Settings
notifications:
  # Email notifications
  email:
    enabled: true
    smtp:
      host: "smtp.gmail.com"
      port: 587
      username: "${SMTP_USERNAME}"
      password: "${SMTP_PASSWORD}"
      from: "terraback@example.com"
      tls: true

  # Notification triggers
  triggers:
    backup_success: true
    backup_failure: true
    drift_detected: true
    restore_completed: true
    storage_threshold: true

  # Rate limiting
  rate_limit:
    max_per_hour: 10
    batch_window: "5m"

# Integration Settings
integrations:
  # Git integration
  git:
    enabled: true
    auto_commit: true
    branch: "terraback-states"
    remote: "origin"
    commit_message: "Automated backup: {timestamp}"
    push_on_backup: true

  # CI/CD integrations
  ci_cd:
    jenkins:
      enabled: false
      url: "https://jenkins.example.com"
      token: "${JENKINS_TOKEN}"
    github_actions:
      enabled: true
      workflow_dispatch: true
    gitlab_ci:
      enabled: false
      project_id: "123"
      token: "${GITLAB_TOKEN}"

  # Monitoring integrations
  monitoring:
    datadog:
      enabled: false
      api_key: "${DATADOG_API_KEY}"
      site: "datadoghq.com"
    new_relic:
      enabled: false
      license_key: "${NEW_RELIC_LICENSE}"
    splunk:
      enabled: false
      hec_endpoint: "https://splunk.example.com:8088"
      hec_token: "${SPLUNK_HEC_TOKEN}"

# Advanced Settings
advanced:
  # Experimental features
  experimental:
    enable_beta_features: false
    ai_drift_analysis: false
    predictive_backup: false

  # Performance tuning
  performance:
    cache_size: "100MB"
    connection_pool_size: 10
    worker_threads: 4

  # Developer settings
  developer:
    debug_mode: false
    verbose_logging: false
    trace_requests: false
    profile_performance: false

Environment Variables

Complete List of Environment Variables

# Core Configuration
export TERRABACK_CONFIG_PATH="/custom/path/config.yml"
export TERRABACK_PROFILE="production"
export TERRABACK_LOG_LEVEL="info"
export TERRABACK_COLOR_OUTPUT="true"

# API Configuration
export TERRABACK_API_KEY="your-api-key-here"
export TERRABACK_API_URL="https://api.terraback.io"
export TERRABACK_API_TIMEOUT="30s"

# Storage Configuration
export TERRABACK_STORAGE_TYPE="s3"
export TERRABACK_STORAGE_BUCKET="my-backups"
export TERRABACK_STORAGE_REGION="us-east-1"
export TERRABACK_STORAGE_PREFIX="terraform/"
export TERRABACK_STORAGE_ENCRYPT="true"

# AWS Specific
export AWS_ACCESS_KEY_ID="AKIA..."
export AWS_SECRET_ACCESS_KEY="..."
export AWS_SESSION_TOKEN="..."
export AWS_REGION="us-east-1"

# Azure Specific
export AZURE_STORAGE_ACCOUNT="terrabackstorage"
export AZURE_STORAGE_ACCESS_KEY="..."
export AZURE_STORAGE_SAS_TOKEN="..."

# GCP Specific
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/credentials.json"
export GCP_PROJECT="my-project"

# Backup Configuration
export TERRABACK_BACKUP_AUTO="true"
export TERRABACK_BACKUP_INTERVAL="1h"
export TERRABACK_BACKUP_RETENTION_DAYS="30"
export TERRABACK_BACKUP_COMPRESSION="true"
export TERRABACK_BACKUP_ENCRYPTION="true"

# Drift Configuration
export TERRABACK_DRIFT_ENABLED="true"
export TERRABACK_DRIFT_INTERVAL="15m"
export TERRABACK_DRIFT_ALERT_EMAIL="ops@example.com"
export TERRABACK_DRIFT_AUTO_FIX="false"

# Network Configuration
export HTTP_PROXY="http://proxy.example.com:8080"
export HTTPS_PROXY="http://proxy.example.com:8080"
export NO_PROXY="localhost,127.0.0.1,.example.com"

# Security Configuration
export TERRABACK_REQUIRE_MFA="true"
export TERRABACK_ENCRYPTION_KEY="/path/to/key"
export TERRABACK_AUDIT_LOG="/var/log/terraback/audit.log"

# Notification Configuration
export TERRABACK_SMTP_HOST="smtp.gmail.com"
export TERRABACK_SMTP_PORT="587"
export TERRABACK_SMTP_USERNAME="user@example.com"
export TERRABACK_SMTP_PASSWORD="password"
export TERRABACK_SLACK_WEBHOOK="https://hooks.slack.com/..."

# Feature Flags
export TERRABACK_BETA_FEATURES="true"
export TERRABACK_AI_ANALYSIS="false"
export TERRABACK_PREDICTIVE_BACKUP="false"

Profile Management

Creating Profiles

# Create development profile
terraback profile create dev \
  --storage-type local \
  --path ~/.terraback/dev-backups \
  --retention 7d

# Create staging profile
terraback profile create staging \
  --storage-type s3 \
  --bucket staging-backups \
  --retention 14d \
  --compression true

# Create production profile
terraback profile create production \
  --storage-type s3 \
  --bucket prod-backups \
  --retention 90d \
  --encryption true \
  --drift-detection true

Using Profiles

# List available profiles
terraback profile list

# Switch to profile
terraback profile use staging

# Use profile for single command
terraback backup create --profile production

# Show current profile
terraback profile current

# Show profile configuration
terraback profile show staging

Profile Configuration Files

# ~/.terraback/profiles/development.yml
name: development
storage:
  type: local
  path: ~/.terraback/dev-backups
backup:
  retention:
    days: 7
  compression:
    enabled: false
drift:
  enabled: false

# ~/.terraback/profiles/production.yml
name: production
storage:
  type: s3
  bucket: prod-terraback-backups
  region: us-east-1
backup:
  retention:
    days: 90
  encryption:
    enabled: true
drift:
  enabled: true
  interval: 5m
  alerts:
    enabled: true

Storage Configuration

S3 Configuration

storage:
  type: s3
  s3:
    # Basic configuration
    bucket: "my-terraback-backups"
    region: "us-east-1"
    prefix: "terraform-states/"

    # Advanced options
    storage_class: "INTELLIGENT_TIERING"  # STANDARD, STANDARD_IA, GLACIER
    server_side_encryption: "aws:kms"
    kms_key_id: "arn:aws:kms:us-east-1:123456789012:key/12345678-1234"

    # Lifecycle policies
    lifecycle:
      transition_to_ia: 30  # days
      transition_to_glacier: 90  # days
      expiration: 365  # days

    # S3-compatible services (MinIO, Wasabi, etc.)
    endpoint: "https://s3.wasabi.com"
    force_path_style: true
    disable_ssl: false
    s3_force_region: "us-east-1"

Azure Blob Storage Configuration

storage:
  type: azure
  azure:
    # Authentication methods (choose one)
    # Method 1: Access Key
    account_name: "terrabackstorage"
    access_key: "${AZURE_STORAGE_ACCESS_KEY}"

    # Method 2: SAS Token
    sas_token: "${AZURE_STORAGE_SAS_TOKEN}"

    # Method 3: Connection String
    connection_string: "${AZURE_STORAGE_CONNECTION_STRING}"

    # Container configuration
    container_name: "terraback-backups"

    # Advanced options
    tier: "Hot"  # Hot, Cool, Archive
    redundancy: "LRS"  # LRS, GRS, RAGRS, ZRS

    # Lifecycle management
    lifecycle:
      transition_to_cool: 30  # days
      transition_to_archive: 90  # days
      delete_after: 365  # days

Google Cloud Storage Configuration

storage:
  type: gcp
  gcp:
    # Authentication
    credentials_path: "/path/to/service-account.json"
    # Or use Application Default Credentials
    use_default_credentials: true

    # Bucket configuration
    bucket: "terraback-backups"
    project: "my-project-id"
    location: "us-central1"

    # Storage options
    storage_class: "STANDARD"  # STANDARD, NEARLINE, COLDLINE, ARCHIVE

    # Lifecycle policies
    lifecycle:
      transition_to_nearline: 30  # days
      transition_to_coldline: 90  # days
      transition_to_archive: 365  # days
      delete_after: 730  # days

    # Advanced options
    uniform_bucket_level_access: true
    versioning: true

Local Storage Configuration

storage:
  type: local
  local:
    # Path configuration
    path: "~/.terraback/backups"  # Supports ~ expansion
    # or absolute path
    path: "/mnt/backup-drive/terraback"

    # Storage management
    max_size: "100GB"  # Maximum storage to use
    auto_cleanup: true  # Auto-delete old backups when full

    # Organization
    structure: "date-based"  # date-based, project-based, flat
    date_format: "%Y/%m/%d"

    # Performance
    compress: true
    compression_level: 6
    parallel_writes: 4

    # Security
    encrypt: true
    encryption_key_file: "~/.terraback/encryption.key"
    permissions: "0600"  # File permissions for backups

Security Configuration

Encryption Settings

security:
  encryption:
    # At-rest encryption
    at_rest:
      enabled: true
      algorithm: "AES256-GCM"
      key_management: "aws-kms"  # aws-kms, azure-keyvault, gcp-kms, local
      key_rotation: true
      rotation_period: "90d"

    # In-transit encryption
    in_transit:
      enabled: true
      tls_version: "1.3"
      cipher_suites:
        - "TLS_AES_128_GCM_SHA256"
        - "TLS_AES_256_GCM_SHA384"

    # Key management
    keys:
      master_key_path: "/secure/location/master.key"
      derive_keys: true
      key_derivation_function: "PBKDF2"
      iterations: 100000

Access Control

security:
  access_control:
    # Authentication methods
    authentication:
      methods:
        - type: "api_key"
          enabled: true
        - type: "oauth"
          enabled: true
          providers: ["github", "google", "okta"]
        - type: "saml"
          enabled: true
          idp_url: "https://idp.example.com"

    # Role-based access control
    rbac:
      enabled: true
      roles:
        - name: "admin"
          permissions:
            - "backup:*"
            - "restore:*"
            - "config:*"
            - "drift:*"
        - name: "operator"
          permissions:
            - "backup:create"
            - "backup:list"
            - "restore:preview"
            - "drift:view"
        - name: "viewer"
          permissions:
            - "backup:list"
            - "backup:show"
            - "drift:view"

    # User mappings
    users:
      - email: "admin@example.com"
        role: "admin"
      - email: "ops-team@example.com"
        role: "operator"
      - email: "*@example.com"  # Wildcard
        role: "viewer"

Advanced Configuration

Performance Tuning

advanced:
  performance:
    # Memory settings
    memory:
      max_heap: "2GB"
      buffer_size: "64MB"
      cache_size: "512MB"

    # Concurrency
    concurrency:
      worker_threads: 8
      io_threads: 4
      max_parallel_operations: 10

    # Network optimization
    network:
      connection_pool_size: 20
      keep_alive: true
      tcp_nodelay: true
      socket_buffer_size: "256KB"

    # Storage optimization
    storage:
      chunk_size: "16MB"
      multipart_threshold: "100MB"
      parallel_uploads: 5
      parallel_downloads: 5

    # Caching
    cache:
      enabled: true
      type: "memory"  # memory, redis, disk
      ttl: "1h"
      max_entries: 1000

Monitoring & Observability

monitoring:
  # Metrics collection
  metrics:
    enabled: true
    collectors:
      - type: "prometheus"
        endpoint: "/metrics"
        port: 9090
      - type: "statsd"
        host: "localhost"
        port: 8125

    # Custom metrics
    custom:
      - name: "backup_duration_seconds"
        type: "histogram"
        buckets: [0.1, 0.5, 1, 5, 10, 30, 60]
      - name: "storage_usage_bytes"
        type: "gauge"

  # Distributed tracing
  tracing:
    enabled: true
    provider: "jaeger"
    endpoint: "http://jaeger:14268"
    sample_rate: 0.1

  # Logging
  logging:
    level: "info"
    outputs:
      - type: "console"
        format: "json"
      - type: "file"
        path: "/var/log/terraback/app.log"
        rotation: "daily"
        retention: "30d"
      - type: "syslog"
        address: "localhost:514"
        facility: "local0"

Validation

Validate Configuration

# Validate current configuration
terraback config validate

# Validate specific file
terraback config validate --file /path/to/config.yml

# Validate with verbose output
terraback config validate --verbose

# Test configuration
terraback config test

Configuration Health Check

# Run configuration diagnostics
terraback doctor config

# Check specific section
terraback doctor config --section storage

# Auto-fix common issues
terraback doctor config --fix

Migration

Migrating Configuration

# Export current configuration
terraback config export > config-backup.yml

# Import configuration
terraback config import config-backup.yml

# Migrate from v1 to v2 format
terraback config migrate --from v1 --to v2

# Convert between formats
terraback config convert --from yaml --to json

Troubleshooting Configuration

Common Issues

1. Configuration Not Loading

# Check configuration search path
terraback config paths

# Verify configuration syntax
terraback config validate --strict

2. Environment Variables Not Working

# Show resolved configuration
terraback config show --resolved

# Debug environment variables
terraback config debug --env

3. Profile Issues

# Reset to default profile
terraback profile reset

# Repair profile
terraback profile repair production