add airbyte docker compose

2026-02-17 09:00:21 +07:00
parent dbc52384ba
commit cdd27e1f5e
5 changed files with 196 additions and 28 deletions
--- a/04-ingestion/README.md
+++ b/04-ingestion/README.md
@@ -1,10 +1,14 @@
 # 04-ingestion: Airbyte Data Ingestion

-Airbyte OSS for data ingestion and ETL (standalone deployment).
+Airbyte OSS for data ingestion and ETL (multi-container deployment).

 ## Services

- **airbyte**: All-in-one Airbyte container (webapp + server + worker + temporal)
+- **airbyte-proxy**: Public entrypoint (UI/API gateway)
+- **server**: Airbyte backend
+- **worker**: Runs sync jobs and launches connector containers
+- **webapp**: Airbyte UI
+- **airbyte-temporal**: Workflow engine

 ## Run

@@ -15,14 +19,11 @@ docker compose --env-file ../.env.global up -d
 ## Access

 - Web UI: http://localhost:8000
- Configure in Nginx to route domain to `airbyte:8000`
+- Configure in Nginx to route domain to `airbyte-proxy:8000`

 ## Note

-This uses Airbyte standalone image for simplicity. For production with high workload:
- Consider using full Airbyte stack with separate containers
- Or use `abctl` (Airbyte Command Line Tool)
- See: https://docs.airbyte.com/deploying-airbyte/docker-compose
+This deployment pins Airbyte images to avoid `:latest` tag issues.

 ## First Time Setup
 1. Create database: `docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"`
--- a/04-ingestion/docker-compose.yml
+++ b/04-ingestion/docker-compose.yml
@@ -1,27 +1,192 @@
-services:
-  # Airbyte OSS - Simplified deployment
-  # Note: For production, consider using abctl or full Airbyte stack
-  # This is a minimal setup for development/testing
+# services:
+#   # Airbyte OSS - Simplified deployment
+#   # Note: For production, consider using abctl or full Airbyte stack
+#   # This is a minimal setup for development/testing
  
-  airbyte:
-    image: airbyte/airbyte-standalone:0.50.33
-    container_name: airbyte
-    ports:
-      - "8000:8000"
-    environment:
-      - DATABASE_HOST=postgres
-      - DATABASE_PORT=5432
-      - DATABASE_USER=${DB_USER}
-      - DATABASE_PASSWORD=${DB_PASSWORD}
-      - DATABASE_DB=airbyte
-      - TZ=${TZ:-Asia/Bangkok}
+#   airbyte:
+#     image: airbyte/airbyte-standalone:0.50.33
+#     container_name: airbyte
+#     ports:
+#       - "8000:8000"
+#     environment:
+#       - DATABASE_HOST=postgres
+#       - DATABASE_PORT=5432
+#       - DATABASE_USER=${DB_USER}
+#       - DATABASE_PASSWORD=${DB_PASSWORD}
+#       - DATABASE_DB=airbyte
+#       - TZ=${TZ:-Asia/Bangkok}
+#     volumes:
+#       - ./data/workspace:/workspace
+#       - ./data/config:/config
+#       - /var/run/docker.sock:/var/run/docker.sock
+#     networks:
+#       - shared_data_network
+#     restart: unless-stopped
+
+# networks:
+#   shared_data_network:
+#     external: true
+
+services:
+  docker-proxy:
+    image: alpine/socat
+    container_name: airbyte-docker-proxy
+    command: -t 900 TCP-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock
+    restart: unless-stopped
+    user: root
    volumes:
-      - ./data/workspace:/workspace
-      - ./data/config:/config
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - shared_data_network
+
+  airbyte-temporal:
+    image: airbyte/temporal:0.63.8
+    container_name: airbyte-temporal
    restart: unless-stopped
+    environment:
+      - DB=postgresql
+      - DB_PORT=${DB_PORT}
+      - POSTGRES_DB=temporal
+      - VISIBILITY_POSTGRES_DB=temporal_visibility
+      - POSTGRES_SEEDS=${DB_HOST}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_PWD=${DB_PASSWORD}
+    networks:
+      - shared_data_network
+
+  bootloader:
+    image: airbyte/bootloader:0.63.8
+    container_name: airbyte-bootloader
+    restart: "no"
+    environment:
+      - AIRBYTE_VERSION=0.63.8
+      - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - LOG_LEVEL=INFO
+    networks:
+      - shared_data_network
+
+  worker:
+    image: airbyte/worker:0.63.8
+    container_name: airbyte-worker
+    restart: unless-stopped
+    depends_on:
+      - docker-proxy
+      - airbyte-temporal
+      - bootloader
+    environment:
+      - AIRBYTE_VERSION=0.63.8
+      - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - CONFIG_ROOT=/data
+      - WORKSPACE_ROOT=/workspace
+      - LOCAL_ROOT=/local_root
+      - TEMPORAL_HOST=airbyte-temporal:7233
+      - LOG_LEVEL=INFO
+      - WORKER_ENVIRONMENT=docker
+      - DOCKER_HOST=docker-proxy:2375
+    volumes:
+      - ./data/workspace:/workspace
+      - ./data/config:/data
+      - ./data/local_root:/local_root
+    networks:
+      - shared_data_network
+
+  server:
+    image: airbyte/server:0.63.8
+    container_name: airbyte-server
+    restart: unless-stopped
+    depends_on:
+      - airbyte-temporal
+      - bootloader
+    environment:
+      - AIRBYTE_VERSION=0.63.8
+      - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - CONFIG_ROOT=/data
+      - WORKSPACE_ROOT=/workspace
+      - LOCAL_ROOT=/local_root
+      - TEMPORAL_HOST=airbyte-temporal:7233
+      - WEBAPP_URL=http://localhost:8000
+      - LOG_LEVEL=INFO
+      - WORKER_ENVIRONMENT=docker
+    volumes:
+      - ./data/workspace:/workspace
+      - ./data/config:/data
+      - ./data/local_root:/local_root
+    networks:
+      - shared_data_network
+
+  webapp:
+    image: airbyte/webapp:0.63.8
+    container_name: airbyte-webapp
+    restart: unless-stopped
+    environment:
+      - AIRBYTE_SERVER_HOST=server:8001
+    networks:
+      - shared_data_network
+    depends_on:
+      - server
+
+  airbyte-cron:
+    image: airbyte/cron:0.63.8
+    container_name: airbyte-cron
+    restart: unless-stopped
+    depends_on:
+      - bootloader
+    environment:
+      - AIRBYTE_VERSION=0.63.8
+      - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - WORKSPACE_ROOT=/workspace
+      - TEMPORAL_HOST=airbyte-temporal:7233
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./data/workspace:/workspace
+    networks:
+      - shared_data_network
+
+  airbyte-api-server:
+    image: airbyte/airbyte-api-server:0.63.8
+    container_name: airbyte-api-server
+    restart: unless-stopped
+    depends_on:
+      - bootloader
+    networks:
+      - shared_data_network
+
+  airbyte-connector-builder-server:
+    image: airbyte/connector-builder-server:0.63.8
+    container_name: airbyte-connector-builder-server
+    restart: unless-stopped
+    depends_on:
+      - bootloader
+    networks:
+      - shared_data_network
+
+  airbyte-proxy:
+    image: airbyte/proxy:0.63.8
+    container_name: airbyte-proxy
+    restart: unless-stopped
+    ports:
+      - "8000:8000"
+      - "8001:8001"
+      - "8003:8003"
+      - "8006:8006"
+    environment:
+      - BASIC_AUTH_USERNAME=${AIRBYTE_BASIC_AUTH_USERNAME:-}
+      - BASIC_AUTH_PASSWORD=${AIRBYTE_BASIC_AUTH_PASSWORD:-}
+      - BASIC_AUTH_PROXY_TIMEOUT=${AIRBYTE_BASIC_AUTH_PROXY_TIMEOUT:-}
+    networks:
+      - shared_data_network
+    depends_on:
+      - webapp
+      - server
+      - airbyte-api-server

 networks:
  shared_data_network:
--- a/DEPLOYMENT.md
+++ b/DEPLOYMENT.md
@@ -111,7 +111,7 @@ You should see:

 **Airbyte:**
 - Domain: `etl.sriphat.local`
- Forward Hostname: `airbyte`
+- Forward Hostname: `airbyte-proxy`
 - Forward Port: `8000`

 ### 2. Setup Keycloak SSO
--- a/setup-ubuntu.sh
+++ b/setup-ubuntu.sh
@@ -56,7 +56,7 @@ fi

 # Install Docker Compose (standalone - backup)
 echo "[3/6] Installing Docker Compose standalone..."
-if ! command -v docker-compose &> /dev/null; then
+if ! docker compose version &> /dev/null && ! command -v docker-compose &> /dev/null; then
    DOCKER_COMPOSE_VERSION=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep 'tag_name' | cut -d\" -f4)
    curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
    chmod +x /usr/local/bin/docker-compose
--- a/start-all.sh
+++ b/start-all.sh
@@ -22,6 +22,8 @@ sleep 10

 echo "[3/7] Creating databases for Airbyte and Superset..."
 docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'airbyte'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"
+docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'temporal'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE temporal;"
+docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'temporal_visibility'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE temporal_visibility;"
 docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'superset'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE superset;"

 echo "[4/7] Starting API Service..."
@@ -50,7 +52,7 @@ echo "- API Service: Configure in Nginx (apiservice:8000)"
 echo "- Superset: Configure in Nginx (superset:8088)"
 echo ""
 echo "Next steps:"
-echo "1. Configure domains in Nginx Proxy Manager (port 81)"
+echo "1. Configure domains in Nginx Proxy Manager (port 8021)"
 echo "2. Setup Keycloak realm and clients"
 echo "3. Configure Airbyte sources/destinations"
 echo "4. Setup Superset dashboards"