229 lines
7.6 KiB
Bash
229 lines
7.6 KiB
Bash
#!/bin/bash
|
|
|
|
# Airbyte OSS Setup Script
|
|
# This script automates the installation of Airbyte using abctl
|
|
# with configuration to use shared PostgreSQL from 01-infra
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
ENV_FILE="$SCRIPT_DIR/.airbyte.env"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Load configuration
|
|
if [ -f "$ENV_FILE" ]; then
|
|
source "$ENV_FILE"
|
|
else
|
|
echo -e "${RED}Error: Configuration file .airbyte.env not found${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
echo -e "${GREEN}=== Airbyte OSS Setup ===${NC}"
|
|
echo "This script will install Airbyte using abctl with shared PostgreSQL"
|
|
echo ""
|
|
|
|
# Check prerequisites
|
|
echo -e "${YELLOW}Checking prerequisites...${NC}"
|
|
|
|
# Check Docker
|
|
if ! command -v docker &> /dev/null; then
|
|
echo -e "${RED}Error: Docker is not installed${NC}"
|
|
echo "Please install Docker Desktop first"
|
|
exit 1
|
|
fi
|
|
|
|
# Check if Docker is running
|
|
if ! docker info &> /dev/null; then
|
|
echo -e "${RED}Error: Docker is not running${NC}"
|
|
echo "Please start Docker Desktop"
|
|
exit 1
|
|
fi
|
|
|
|
echo -e "${GREEN}✓ Docker is installed and running${NC}"
|
|
|
|
# Check if PostgreSQL container is running
|
|
if ! docker ps | grep -q "postgres"; then
|
|
echo -e "${RED}Error: PostgreSQL container is not running${NC}"
|
|
echo "Please start the infrastructure first:"
|
|
echo " cd $PROJECT_ROOT/01-infra"
|
|
echo " docker compose --env-file ../.env.global up -d"
|
|
exit 1
|
|
fi
|
|
|
|
echo -e "${GREEN}✓ PostgreSQL container is running${NC}"
|
|
|
|
# Install abctl if not present
|
|
echo -e "${YELLOW}Checking for abctl...${NC}"
|
|
if ! command -v abctl &> /dev/null; then
|
|
echo "abctl not found. Installing..."
|
|
|
|
# Detect OS
|
|
OS="$(uname -s)"
|
|
case "${OS}" in
|
|
Linux*)
|
|
curl -LsfS https://get.airbyte.com | bash -
|
|
;;
|
|
Darwin*)
|
|
curl -LsfS https://get.airbyte.com | bash -
|
|
;;
|
|
MINGW*|MSYS*|CYGWIN*)
|
|
echo -e "${RED}Windows detected. Please install abctl manually:${NC}"
|
|
echo "Download from: https://github.com/airbytehq/abctl/releases"
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo -e "${RED}Unsupported OS: ${OS}${NC}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
# Verify installation
|
|
if ! command -v abctl &> /dev/null; then
|
|
echo -e "${RED}Failed to install abctl${NC}"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo -e "${GREEN}✓ abctl is installed${NC}"
|
|
|
|
# Create databases in PostgreSQL
|
|
echo -e "${YELLOW}Creating Airbyte databases...${NC}"
|
|
|
|
# Check if databases already exist
|
|
DB_EXISTS=$(docker exec postgres psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='airbyte'" 2>/dev/null || echo "0")
|
|
|
|
if [ "$DB_EXISTS" = "1" ]; then
|
|
echo -e "${YELLOW}Airbyte databases already exist. Skipping creation.${NC}"
|
|
else
|
|
echo "Creating airbyte, temporal, and temporal_visibility databases..."
|
|
docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;" 2>/dev/null || true
|
|
docker exec postgres psql -U postgres -c "CREATE DATABASE temporal;" 2>/dev/null || true
|
|
docker exec postgres psql -U postgres -c "CREATE DATABASE temporal_visibility;" 2>/dev/null || true
|
|
docker exec postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE airbyte TO postgres;" 2>/dev/null || true
|
|
docker exec postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE temporal TO postgres;" 2>/dev/null || true
|
|
docker exec postgres psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE temporal_visibility TO postgres;" 2>/dev/null || true
|
|
echo -e "${GREEN}✓ Databases created${NC}"
|
|
fi
|
|
|
|
# Prepare abctl install command
|
|
echo -e "${YELLOW}Preparing Airbyte installation...${NC}"
|
|
echo "Configuration:"
|
|
echo " - Version: Latest stable (determined by abctl)"
|
|
echo " - Port: ${AIRBYTE_PORT}"
|
|
echo " - Domain: ${AIRBYTE_HOST}"
|
|
echo " - Low Resource Mode: ${LOW_RESOURCE_MODE}"
|
|
echo " - Backup: ${ENABLE_BACKUP}"
|
|
echo ""
|
|
|
|
INSTALL_CMD="abctl local install"
|
|
|
|
# Add port mapping
|
|
if [ ! -z "$AIRBYTE_PORT" ] && [ "$AIRBYTE_PORT" != "8000" ]; then
|
|
INSTALL_CMD="$INSTALL_CMD --port $AIRBYTE_PORT"
|
|
fi
|
|
|
|
# Add host if specified (for domain access)
|
|
if [ ! -z "$AIRBYTE_HOST" ]; then
|
|
INSTALL_CMD="$INSTALL_CMD --host $AIRBYTE_HOST"
|
|
fi
|
|
|
|
# Add insecure cookies flag (required when behind nginx proxy)
|
|
if [ "$INSECURE_COOKIES" = "true" ]; then
|
|
INSTALL_CMD="$INSTALL_CMD --insecure-cookies"
|
|
fi
|
|
|
|
# Add low resource mode (enabled by default)
|
|
if [ "$LOW_RESOURCE_MODE" = "true" ]; then
|
|
INSTALL_CMD="$INSTALL_CMD --low-resource-mode"
|
|
echo -e "${YELLOW}Note: Low-resource mode enabled. Connector Builder will be disabled.${NC}"
|
|
fi
|
|
|
|
echo "Installation command: $INSTALL_CMD"
|
|
echo ""
|
|
|
|
# Run installation
|
|
echo -e "${YELLOW}Installing Airbyte...${NC}"
|
|
echo "This may take up to 30 minutes depending on your internet connection."
|
|
echo ""
|
|
|
|
eval $INSTALL_CMD
|
|
|
|
# Check installation status
|
|
if [ $? -eq 0 ]; then
|
|
echo ""
|
|
echo -e "${GREEN}=== Airbyte Installation Complete ===${NC}"
|
|
echo ""
|
|
|
|
# Setup backup if enabled
|
|
if [ "$ENABLE_BACKUP" = "true" ]; then
|
|
echo -e "${YELLOW}Setting up automated backups...${NC}"
|
|
cat > "$SCRIPT_DIR/backup-airbyte.sh" << 'BACKUP_SCRIPT'
|
|
#!/bin/bash
|
|
# Airbyte Backup Script
|
|
# Backs up Airbyte databases from PostgreSQL
|
|
|
|
BACKUP_DIR="./backups"
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
DATE=$(date +%Y%m%d_%H%M%S)
|
|
|
|
echo "Backing up Airbyte databases..."
|
|
docker exec postgres pg_dump -U postgres airbyte > "$BACKUP_DIR/airbyte_$DATE.sql"
|
|
docker exec postgres pg_dump -U postgres temporal > "$BACKUP_DIR/temporal_$DATE.sql"
|
|
docker exec postgres pg_dump -U postgres temporal_visibility > "$BACKUP_DIR/temporal_visibility_$DATE.sql"
|
|
|
|
# Compress backups
|
|
tar -czf "$BACKUP_DIR/airbyte_backup_$DATE.tar.gz" "$BACKUP_DIR/airbyte_$DATE.sql" "$BACKUP_DIR/temporal_$DATE.sql" "$BACKUP_DIR/temporal_visibility_$DATE.sql"
|
|
rm "$BACKUP_DIR/airbyte_$DATE.sql" "$BACKUP_DIR/temporal_$DATE.sql" "$BACKUP_DIR/temporal_visibility_$DATE.sql"
|
|
|
|
# Keep only last 7 days of backups
|
|
find "$BACKUP_DIR" -name "airbyte_backup_*.tar.gz" -mtime +7 -delete
|
|
|
|
echo "Backup completed: airbyte_backup_$DATE.tar.gz"
|
|
BACKUP_SCRIPT
|
|
chmod +x "$SCRIPT_DIR/backup-airbyte.sh"
|
|
echo -e "${GREEN}✓ Backup script created: backup-airbyte.sh${NC}"
|
|
echo " Run manually: ./backup-airbyte.sh"
|
|
echo " Schedule: Add to crontab with schedule: $BACKUP_SCHEDULE"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Access Airbyte at:"
|
|
echo " Domain: https://ai.sriphat.com/airbyte (via Nginx Proxy Manager)"
|
|
echo " Local: http://localhost:${AIRBYTE_PORT:-8000}"
|
|
echo " Direct: http://[SERVER_IP]:${AIRBYTE_PORT:-8000}"
|
|
echo ""
|
|
echo -e "${YELLOW}Important: Configure Nginx Proxy Manager${NC}"
|
|
echo "1. Go to Nginx Proxy Manager (http://localhost:8021)"
|
|
echo "2. Add Proxy Host:"
|
|
echo " - Domain: ai.sriphat.com"
|
|
echo " - Scheme: http"
|
|
echo " - Forward Hostname: airbyte-proxy"
|
|
echo " - Forward Port: 8000"
|
|
echo " - Custom Location: /airbyte"
|
|
echo "3. Enable SSL certificate"
|
|
echo "4. (Optional) Add Keycloak authentication via nginx"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo "1. Configure Nginx Proxy Manager (see above)"
|
|
echo "2. Open Airbyte in your browser"
|
|
echo "3. Enter your email and organization name"
|
|
echo "4. Configure your sources and destinations"
|
|
echo ""
|
|
echo "To manage Airbyte:"
|
|
echo " Start: ./start-airbyte.sh"
|
|
echo " Stop: ./stop-airbyte.sh"
|
|
echo " Backup: ./backup-airbyte.sh"
|
|
echo " Uninstall: ./uninstall-airbyte.sh"
|
|
echo ""
|
|
else
|
|
echo -e "${RED}Installation failed. Please check the error messages above.${NC}"
|
|
exit 1
|
|
fi
|