add superset airbyte setup and merge md file

04-ingestion/setup-airbyte.sh

@@ -0,0 +1,228 @@
+#!/bin/bash
+
+# Airbyte OSS Setup Script
+# This script automates the installation of Airbyte using abctl
+# with configuration to use shared PostgreSQL from 01-infra
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+ENV_FILE="$SCRIPT_DIR/.airbyte.env"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Load configuration
+if [ -f "$ENV_FILE" ]; then
+    source "$ENV_FILE"
+else
+    echo -e "${RED}Error: Configuration file .airbyte.env not found${NC}"
+    exit 1
+fi
+
+echo -e "${GREEN}=== Airbyte OSS Setup ===${NC}"
+echo "This script will install Airbyte using abctl with shared PostgreSQL"
+echo ""
+
+# Check prerequisites
+echo -e "${YELLOW}Checking prerequisites...${NC}"
+
+# Check Docker
+if ! command -v docker &> /dev/null; then
+    echo -e "${RED}Error: Docker is not installed${NC}"
+    echo "Please install Docker Desktop first"
+    exit 1
+fi
+
+# Check if Docker is running
+if ! docker info &> /dev/null; then
+    echo -e "${RED}Error: Docker is not running${NC}"
+    echo "Please start Docker Desktop"
+    exit 1
+fi
+
+echo -e "${GREEN}✓ Docker is installed and running${NC}"
+
+# Check if PostgreSQL container is running
+if ! docker ps | grep -q "postgres"; then
+    echo -e "${RED}Error: PostgreSQL container is not running${NC}"
+    echo "Please start the infrastructure first:"
+    echo "  cd $PROJECT_ROOT/01-infra"
+    echo "  docker compose --env-file ../.env.global up -d"
+    exit 1
+fi
+
+echo -e "${GREEN}✓ PostgreSQL container is running${NC}"
+
+# Install abctl if not present
+echo -e "${YELLOW}Checking for abctl...${NC}"
+if ! command -v abctl &> /dev/null; then
+    echo "abctl not found. Installing..."
+    
+    # Detect OS
+    OS="$(uname -s)"
+    case "${OS}" in
+        Linux*)     
+            curl -LsfS https://get.airbyte.com | bash -
+            ;;
+        Darwin*)    
+            curl -LsfS https://get.airbyte.com | bash -
+            ;;
+        MINGW*|MSYS*|CYGWIN*)
+            echo -e "${RED}Windows detected. Please install abctl manually:${NC}"
+            echo "Download from: https://github.com/airbytehq/abctl/releases"
+            exit 1
+            ;;
+        *)
+            echo -e "${RED}Unsupported OS: ${OS}${NC}"
+            exit 1
+            ;;
+    esac
+    
+    # Verify installation
+    if ! command -v abctl &> /dev/null; then
+        echo -e "${RED}Failed to install abctl${NC}"
+        exit 1
+    fi
+fi
+
+echo -e "${GREEN}✓ abctl is installed${NC}"
+
+# Create databases in PostgreSQL
+echo -e "${YELLOW}Creating Airbyte databases...${NC}"
+
+# Check if databases already exist
+DB_EXISTS=$(docker exec postgres psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='airbyte'" 2>/dev/null || echo "0")
+
+if [ "$DB_EXISTS" = "1" ]; then
+    echo -e "${YELLOW}Airbyte databases already exist. Skipping creation.${NC}"
+else
+    echo "Creating airbyte, temporal, and temporal_visibility databases..."
+    docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;" 2>/dev/null || true
+    docker exec postgres psql -U postgres -c "CREATE DATABASE temporal;" 2>/dev/null || true
+    docker exec postgres psql -U postgres -c "CREATE DATABASE temporal_visibility;" 2>/dev/null || true
+    docker exec postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE airbyte TO postgres;" 2>/dev/null || true
+    docker exec postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE temporal TO postgres;" 2>/dev/null || true
+    docker exec postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE temporal_visibility TO postgres;" 2>/dev/null || true
+    echo -e "${GREEN}✓ Databases created${NC}"
+fi
+
+# Prepare abctl install command
+echo -e "${YELLOW}Preparing Airbyte installation...${NC}"
+echo "Configuration:"
+echo "  - Version: Latest stable (determined by abctl)"
+echo "  - Port: ${AIRBYTE_PORT}"
+echo "  - Domain: ${AIRBYTE_HOST}"
+echo "  - Low Resource Mode: ${LOW_RESOURCE_MODE}"
+echo "  - Backup: ${ENABLE_BACKUP}"
+echo ""
+
+INSTALL_CMD="abctl local install"
+
+# Add port mapping
+if [ ! -z "$AIRBYTE_PORT" ] && [ "$AIRBYTE_PORT" != "8000" ]; then
+    INSTALL_CMD="$INSTALL_CMD --port $AIRBYTE_PORT"
+fi
+
+# Add host if specified (for domain access)
+if [ ! -z "$AIRBYTE_HOST" ]; then
+    INSTALL_CMD="$INSTALL_CMD --host $AIRBYTE_HOST"
+fi
+
+# Add insecure cookies flag (required when behind nginx proxy)
+if [ "$INSECURE_COOKIES" = "true" ]; then
+    INSTALL_CMD="$INSTALL_CMD --insecure-cookies"
+fi
+
+# Add low resource mode (enabled by default)
+if [ "$LOW_RESOURCE_MODE" = "true" ]; then
+    INSTALL_CMD="$INSTALL_CMD --low-resource-mode"
+    echo -e "${YELLOW}Note: Low-resource mode enabled. Connector Builder will be disabled.${NC}"
+fi
+
+echo "Installation command: $INSTALL_CMD"
+echo ""
+
+# Run installation
+echo -e "${YELLOW}Installing Airbyte...${NC}"
+echo "This may take up to 30 minutes depending on your internet connection."
+echo ""
+
+eval $INSTALL_CMD
+
+# Check installation status
+if [ $? -eq 0 ]; then
+    echo ""
+    echo -e "${GREEN}=== Airbyte Installation Complete ===${NC}"
+    echo ""
+    
+    # Setup backup if enabled
+    if [ "$ENABLE_BACKUP" = "true" ]; then
+        echo -e "${YELLOW}Setting up automated backups...${NC}"
+        cat > "$SCRIPT_DIR/backup-airbyte.sh" << 'BACKUP_SCRIPT'
+#!/bin/bash
+# Airbyte Backup Script
+# Backs up Airbyte databases from PostgreSQL
+
+BACKUP_DIR="./backups"
+mkdir -p "$BACKUP_DIR"
+
+DATE=$(date +%Y%m%d_%H%M%S)
+
+echo "Backing up Airbyte databases..."
+docker exec postgres pg_dump -U postgres airbyte > "$BACKUP_DIR/airbyte_$DATE.sql"
+docker exec postgres pg_dump -U postgres temporal > "$BACKUP_DIR/temporal_$DATE.sql"
+docker exec postgres pg_dump -U postgres temporal_visibility > "$BACKUP_DIR/temporal_visibility_$DATE.sql"
+
+# Compress backups
+tar -czf "$BACKUP_DIR/airbyte_backup_$DATE.tar.gz" "$BACKUP_DIR/airbyte_$DATE.sql" "$BACKUP_DIR/temporal_$DATE.sql" "$BACKUP_DIR/temporal_visibility_$DATE.sql"
+rm "$BACKUP_DIR/airbyte_$DATE.sql" "$BACKUP_DIR/temporal_$DATE.sql" "$BACKUP_DIR/temporal_visibility_$DATE.sql"
+
+# Keep only last 7 days of backups
+find "$BACKUP_DIR" -name "airbyte_backup_*.tar.gz" -mtime +7 -delete
+
+echo "Backup completed: airbyte_backup_$DATE.tar.gz"
+BACKUP_SCRIPT
+        chmod +x "$SCRIPT_DIR/backup-airbyte.sh"
+        echo -e "${GREEN}✓ Backup script created: backup-airbyte.sh${NC}"
+        echo "  Run manually: ./backup-airbyte.sh"
+        echo "  Schedule: Add to crontab with schedule: $BACKUP_SCHEDULE"
+    fi
+    
+    echo ""
+    echo "Access Airbyte at:"
+    echo "  Domain: https://ai.sriphat.com/airbyte (via Nginx Proxy Manager)"
+    echo "  Local:  http://localhost:${AIRBYTE_PORT:-8000}"
+    echo "  Direct: http://[SERVER_IP]:${AIRBYTE_PORT:-8000}"
+    echo ""
+    echo -e "${YELLOW}Important: Configure Nginx Proxy Manager${NC}"
+    echo "1. Go to Nginx Proxy Manager (http://localhost:8021)"
+    echo "2. Add Proxy Host:"
+    echo "   - Domain: ai.sriphat.com"
+    echo "   - Scheme: http"
+    echo "   - Forward Hostname: airbyte-proxy"
+    echo "   - Forward Port: 8000"
+    echo "   - Custom Location: /airbyte"
+    echo "3. Enable SSL certificate"
+    echo "4. (Optional) Add Keycloak authentication via nginx"
+    echo ""
+    echo "Next steps:"
+    echo "1. Configure Nginx Proxy Manager (see above)"
+    echo "2. Open Airbyte in your browser"
+    echo "3. Enter your email and organization name"
+    echo "4. Configure your sources and destinations"
+    echo ""
+    echo "To manage Airbyte:"
+    echo "  Start:     ./start-airbyte.sh"
+    echo "  Stop:      ./stop-airbyte.sh"
+    echo "  Backup:    ./backup-airbyte.sh"
+    echo "  Uninstall: ./uninstall-airbyte.sh"
+    echo ""
+else
+    echo -e "${RED}Installation failed. Please check the error messages above.${NC}"
+    exit 1
+fi