From cdd27e1f5e8eb9503f068348ab0d59bb1658b6a6 Mon Sep 17 00:00:00 2001 From: jigoong Date: Tue, 17 Feb 2026 09:00:21 +0700 Subject: [PATCH] add airbyte docker compose --- 04-ingestion/README.md | 15 +-- 04-ingestion/docker-compose.yml | 201 +++++++++++++++++++++++++++++--- DEPLOYMENT.md | 2 +- setup-ubuntu.sh | 2 +- start-all.sh | 4 +- 5 files changed, 196 insertions(+), 28 deletions(-) diff --git a/04-ingestion/README.md b/04-ingestion/README.md index 50ca5bb..ae308d7 100644 --- a/04-ingestion/README.md +++ b/04-ingestion/README.md @@ -1,10 +1,14 @@ # 04-ingestion: Airbyte Data Ingestion -Airbyte OSS for data ingestion and ETL (standalone deployment). +Airbyte OSS for data ingestion and ETL (multi-container deployment). ## Services -- **airbyte**: All-in-one Airbyte container (webapp + server + worker + temporal) +- **airbyte-proxy**: Public entrypoint (UI/API gateway) +- **server**: Airbyte backend +- **worker**: Runs sync jobs and launches connector containers +- **webapp**: Airbyte UI +- **airbyte-temporal**: Workflow engine ## Run @@ -15,14 +19,11 @@ docker compose --env-file ../.env.global up -d ## Access - Web UI: http://localhost:8000 -- Configure in Nginx to route domain to `airbyte:8000` +- Configure in Nginx to route domain to `airbyte-proxy:8000` ## Note -This uses Airbyte standalone image for simplicity. For production with high workload: -- Consider using full Airbyte stack with separate containers -- Or use `abctl` (Airbyte Command Line Tool) -- See: https://docs.airbyte.com/deploying-airbyte/docker-compose +This deployment pins Airbyte images to avoid `:latest` tag issues. ## First Time Setup 1. Create database: `docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"` diff --git a/04-ingestion/docker-compose.yml b/04-ingestion/docker-compose.yml index 5594093..b6d6716 100644 --- a/04-ingestion/docker-compose.yml +++ b/04-ingestion/docker-compose.yml @@ -1,27 +1,192 @@ -services: - # Airbyte OSS - Simplified deployment - # Note: For production, consider using abctl or full Airbyte stack - # This is a minimal setup for development/testing +# services: +# # Airbyte OSS - Simplified deployment +# # Note: For production, consider using abctl or full Airbyte stack +# # This is a minimal setup for development/testing - airbyte: - image: airbyte/airbyte-standalone:0.50.33 - container_name: airbyte - ports: - - "8000:8000" - environment: - - DATABASE_HOST=postgres - - DATABASE_PORT=5432 - - DATABASE_USER=${DB_USER} - - DATABASE_PASSWORD=${DB_PASSWORD} - - DATABASE_DB=airbyte - - TZ=${TZ:-Asia/Bangkok} +# airbyte: +# image: airbyte/airbyte-standalone:0.50.33 +# container_name: airbyte +# ports: +# - "8000:8000" +# environment: +# - DATABASE_HOST=postgres +# - DATABASE_PORT=5432 +# - DATABASE_USER=${DB_USER} +# - DATABASE_PASSWORD=${DB_PASSWORD} +# - DATABASE_DB=airbyte +# - TZ=${TZ:-Asia/Bangkok} +# volumes: +# - ./data/workspace:/workspace +# - ./data/config:/config +# - /var/run/docker.sock:/var/run/docker.sock +# networks: +# - shared_data_network +# restart: unless-stopped + +# networks: +# shared_data_network: +# external: true + +services: + docker-proxy: + image: alpine/socat + container_name: airbyte-docker-proxy + command: -t 900 TCP-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock + restart: unless-stopped + user: root volumes: - - ./data/workspace:/workspace - - ./data/config:/config - /var/run/docker.sock:/var/run/docker.sock networks: - shared_data_network + + airbyte-temporal: + image: airbyte/temporal:0.63.8 + container_name: airbyte-temporal restart: unless-stopped + environment: + - DB=postgresql + - DB_PORT=${DB_PORT} + - POSTGRES_DB=temporal + - VISIBILITY_POSTGRES_DB=temporal_visibility + - POSTGRES_SEEDS=${DB_HOST} + - POSTGRES_USER=${DB_USER} + - POSTGRES_PWD=${DB_PASSWORD} + networks: + - shared_data_network + + bootloader: + image: airbyte/bootloader:0.63.8 + container_name: airbyte-bootloader + restart: "no" + environment: + - AIRBYTE_VERSION=0.63.8 + - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte + - DATABASE_USER=${DB_USER} + - DATABASE_PASSWORD=${DB_PASSWORD} + - LOG_LEVEL=INFO + networks: + - shared_data_network + + worker: + image: airbyte/worker:0.63.8 + container_name: airbyte-worker + restart: unless-stopped + depends_on: + - docker-proxy + - airbyte-temporal + - bootloader + environment: + - AIRBYTE_VERSION=0.63.8 + - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte + - DATABASE_USER=${DB_USER} + - DATABASE_PASSWORD=${DB_PASSWORD} + - CONFIG_ROOT=/data + - WORKSPACE_ROOT=/workspace + - LOCAL_ROOT=/local_root + - TEMPORAL_HOST=airbyte-temporal:7233 + - LOG_LEVEL=INFO + - WORKER_ENVIRONMENT=docker + - DOCKER_HOST=docker-proxy:2375 + volumes: + - ./data/workspace:/workspace + - ./data/config:/data + - ./data/local_root:/local_root + networks: + - shared_data_network + + server: + image: airbyte/server:0.63.8 + container_name: airbyte-server + restart: unless-stopped + depends_on: + - airbyte-temporal + - bootloader + environment: + - AIRBYTE_VERSION=0.63.8 + - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte + - DATABASE_USER=${DB_USER} + - DATABASE_PASSWORD=${DB_PASSWORD} + - CONFIG_ROOT=/data + - WORKSPACE_ROOT=/workspace + - LOCAL_ROOT=/local_root + - TEMPORAL_HOST=airbyte-temporal:7233 + - WEBAPP_URL=http://localhost:8000 + - LOG_LEVEL=INFO + - WORKER_ENVIRONMENT=docker + volumes: + - ./data/workspace:/workspace + - ./data/config:/data + - ./data/local_root:/local_root + networks: + - shared_data_network + + webapp: + image: airbyte/webapp:0.63.8 + container_name: airbyte-webapp + restart: unless-stopped + environment: + - AIRBYTE_SERVER_HOST=server:8001 + networks: + - shared_data_network + depends_on: + - server + + airbyte-cron: + image: airbyte/cron:0.63.8 + container_name: airbyte-cron + restart: unless-stopped + depends_on: + - bootloader + environment: + - AIRBYTE_VERSION=0.63.8 + - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte + - DATABASE_USER=${DB_USER} + - DATABASE_PASSWORD=${DB_PASSWORD} + - WORKSPACE_ROOT=/workspace + - TEMPORAL_HOST=airbyte-temporal:7233 + - LOG_LEVEL=INFO + volumes: + - ./data/workspace:/workspace + networks: + - shared_data_network + + airbyte-api-server: + image: airbyte/airbyte-api-server:0.63.8 + container_name: airbyte-api-server + restart: unless-stopped + depends_on: + - bootloader + networks: + - shared_data_network + + airbyte-connector-builder-server: + image: airbyte/connector-builder-server:0.63.8 + container_name: airbyte-connector-builder-server + restart: unless-stopped + depends_on: + - bootloader + networks: + - shared_data_network + + airbyte-proxy: + image: airbyte/proxy:0.63.8 + container_name: airbyte-proxy + restart: unless-stopped + ports: + - "8000:8000" + - "8001:8001" + - "8003:8003" + - "8006:8006" + environment: + - BASIC_AUTH_USERNAME=${AIRBYTE_BASIC_AUTH_USERNAME:-} + - BASIC_AUTH_PASSWORD=${AIRBYTE_BASIC_AUTH_PASSWORD:-} + - BASIC_AUTH_PROXY_TIMEOUT=${AIRBYTE_BASIC_AUTH_PROXY_TIMEOUT:-} + networks: + - shared_data_network + depends_on: + - webapp + - server + - airbyte-api-server networks: shared_data_network: diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index fc7c00d..00a1543 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -111,7 +111,7 @@ You should see: **Airbyte:** - Domain: `etl.sriphat.local` -- Forward Hostname: `airbyte` +- Forward Hostname: `airbyte-proxy` - Forward Port: `8000` ### 2. Setup Keycloak SSO diff --git a/setup-ubuntu.sh b/setup-ubuntu.sh index aedbaf3..c8e92b3 100644 --- a/setup-ubuntu.sh +++ b/setup-ubuntu.sh @@ -56,7 +56,7 @@ fi # Install Docker Compose (standalone - backup) echo "[3/6] Installing Docker Compose standalone..." -if ! command -v docker-compose &> /dev/null; then +if ! docker compose version &> /dev/null && ! command -v docker-compose &> /dev/null; then DOCKER_COMPOSE_VERSION=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep 'tag_name' | cut -d\" -f4) curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose diff --git a/start-all.sh b/start-all.sh index d873a98..0da6fce 100644 --- a/start-all.sh +++ b/start-all.sh @@ -22,6 +22,8 @@ sleep 10 echo "[3/7] Creating databases for Airbyte and Superset..." docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'airbyte'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;" +docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'temporal'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE temporal;" +docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'temporal_visibility'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE temporal_visibility;" docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'superset'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE superset;" echo "[4/7] Starting API Service..." @@ -50,7 +52,7 @@ echo "- API Service: Configure in Nginx (apiservice:8000)" echo "- Superset: Configure in Nginx (superset:8088)" echo "" echo "Next steps:" -echo "1. Configure domains in Nginx Proxy Manager (port 81)" +echo "1. Configure domains in Nginx Proxy Manager (port 8021)" echo "2. Setup Keycloak realm and clients" echo "3. Configure Airbyte sources/destinations" echo "4. Setup Superset dashboards"