add airbyte docker compose

04-ingestion/README.md

@@ -1,10 +1,14 @@
 # 04-ingestion: Airbyte Data Ingestion
 
-Airbyte OSS for data ingestion and ETL (standalone deployment).
+Airbyte OSS for data ingestion and ETL (multi-container deployment).
 
 ## Services
 
-- **airbyte**: All-in-one Airbyte container (webapp + server + worker + temporal)
+- **airbyte-proxy**: Public entrypoint (UI/API gateway)
+- **server**: Airbyte backend
+- **worker**: Runs sync jobs and launches connector containers
+- **webapp**: Airbyte UI
+- **airbyte-temporal**: Workflow engine
 
 ## Run
 
@@ -15,14 +19,11 @@ docker compose --env-file ../.env.global up -d
 ## Access
 
 - Web UI: http://localhost:8000
-- Configure in Nginx to route domain to `airbyte:8000`
+- Configure in Nginx to route domain to `airbyte-proxy:8000`
 
 ## Note
 
-This uses Airbyte standalone image for simplicity. For production with high workload:
-- Consider using full Airbyte stack with separate containers
-- Or use `abctl` (Airbyte Command Line Tool)
-- See: https://docs.airbyte.com/deploying-airbyte/docker-compose
+This deployment pins Airbyte images to avoid `:latest` tag issues.
 
 ## First Time Setup
 1. Create database: `docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"`

04-ingestion/docker-compose.yml

@@ -1,27 +1,192 @@
-services:
-  # Airbyte OSS - Simplified deployment
-  # Note: For production, consider using abctl or full Airbyte stack
-  # This is a minimal setup for development/testing
+# services:
+#   # Airbyte OSS - Simplified deployment
+#   # Note: For production, consider using abctl or full Airbyte stack
+#   # This is a minimal setup for development/testing
   
-  airbyte:
-    image: airbyte/airbyte-standalone:0.50.33
-    container_name: airbyte
-    ports:
-      - "8000:8000"
-    environment:
-      - DATABASE_HOST=postgres
-      - DATABASE_PORT=5432
-      - DATABASE_USER=${DB_USER}
-      - DATABASE_PASSWORD=${DB_PASSWORD}
-      - DATABASE_DB=airbyte
-      - TZ=${TZ:-Asia/Bangkok}
+#   airbyte:
+#     image: airbyte/airbyte-standalone:0.50.33
+#     container_name: airbyte
+#     ports:
+#       - "8000:8000"
+#     environment:
+#       - DATABASE_HOST=postgres
+#       - DATABASE_PORT=5432
+#       - DATABASE_USER=${DB_USER}
+#       - DATABASE_PASSWORD=${DB_PASSWORD}
+#       - DATABASE_DB=airbyte
+#       - TZ=${TZ:-Asia/Bangkok}
+#     volumes:
+#       - ./data/workspace:/workspace
+#       - ./data/config:/config
+#       - /var/run/docker.sock:/var/run/docker.sock
+#     networks:
+#       - shared_data_network
+#     restart: unless-stopped
+
+# networks:
+#   shared_data_network:
+#     external: true
+
+services:
+  docker-proxy:
+    image: alpine/socat
+    container_name: airbyte-docker-proxy
+    command: -t 900 TCP-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock
+    restart: unless-stopped
+    user: root
     volumes:
-      - ./data/workspace:/workspace
-      - ./data/config:/config
       - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - shared_data_network
+
+  airbyte-temporal:
+    image: airbyte/temporal:0.63.8
+    container_name: airbyte-temporal
     restart: unless-stopped
+    environment:
+      - DB=postgresql
+      - DB_PORT=${DB_PORT}
+      - POSTGRES_DB=temporal
+      - VISIBILITY_POSTGRES_DB=temporal_visibility
+      - POSTGRES_SEEDS=${DB_HOST}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_PWD=${DB_PASSWORD}
+    networks:
+      - shared_data_network
+
+  bootloader:
+    image: airbyte/bootloader:0.63.8
+    container_name: airbyte-bootloader
+    restart: "no"
+    environment:
+      - AIRBYTE_VERSION=0.63.8
+      - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - LOG_LEVEL=INFO
+    networks:
+      - shared_data_network
+
+  worker:
+    image: airbyte/worker:0.63.8
+    container_name: airbyte-worker
+    restart: unless-stopped
+    depends_on:
+      - docker-proxy
+      - airbyte-temporal
+      - bootloader
+    environment:
+      - AIRBYTE_VERSION=0.63.8
+      - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - CONFIG_ROOT=/data
+      - WORKSPACE_ROOT=/workspace
+      - LOCAL_ROOT=/local_root
+      - TEMPORAL_HOST=airbyte-temporal:7233
+      - LOG_LEVEL=INFO
+      - WORKER_ENVIRONMENT=docker
+      - DOCKER_HOST=docker-proxy:2375
+    volumes:
+      - ./data/workspace:/workspace
+      - ./data/config:/data
+      - ./data/local_root:/local_root
+    networks:
+      - shared_data_network
+
+  server:
+    image: airbyte/server:0.63.8
+    container_name: airbyte-server
+    restart: unless-stopped
+    depends_on:
+      - airbyte-temporal
+      - bootloader
+    environment:
+      - AIRBYTE_VERSION=0.63.8
+      - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - CONFIG_ROOT=/data
+      - WORKSPACE_ROOT=/workspace
+      - LOCAL_ROOT=/local_root
+      - TEMPORAL_HOST=airbyte-temporal:7233
+      - WEBAPP_URL=http://localhost:8000
+      - LOG_LEVEL=INFO
+      - WORKER_ENVIRONMENT=docker
+    volumes:
+      - ./data/workspace:/workspace
+      - ./data/config:/data
+      - ./data/local_root:/local_root
+    networks:
+      - shared_data_network
+
+  webapp:
+    image: airbyte/webapp:0.63.8
+    container_name: airbyte-webapp
+    restart: unless-stopped
+    environment:
+      - AIRBYTE_SERVER_HOST=server:8001
+    networks:
+      - shared_data_network
+    depends_on:
+      - server
+
+  airbyte-cron:
+    image: airbyte/cron:0.63.8
+    container_name: airbyte-cron
+    restart: unless-stopped
+    depends_on:
+      - bootloader
+    environment:
+      - AIRBYTE_VERSION=0.63.8
+      - DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - WORKSPACE_ROOT=/workspace
+      - TEMPORAL_HOST=airbyte-temporal:7233
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./data/workspace:/workspace
+    networks:
+      - shared_data_network
+
+  airbyte-api-server:
+    image: airbyte/airbyte-api-server:0.63.8
+    container_name: airbyte-api-server
+    restart: unless-stopped
+    depends_on:
+      - bootloader
+    networks:
+      - shared_data_network
+
+  airbyte-connector-builder-server:
+    image: airbyte/connector-builder-server:0.63.8
+    container_name: airbyte-connector-builder-server
+    restart: unless-stopped
+    depends_on:
+      - bootloader
+    networks:
+      - shared_data_network
+
+  airbyte-proxy:
+    image: airbyte/proxy:0.63.8
+    container_name: airbyte-proxy
+    restart: unless-stopped
+    ports:
+      - "8000:8000"
+      - "8001:8001"
+      - "8003:8003"
+      - "8006:8006"
+    environment:
+      - BASIC_AUTH_USERNAME=${AIRBYTE_BASIC_AUTH_USERNAME:-}
+      - BASIC_AUTH_PASSWORD=${AIRBYTE_BASIC_AUTH_PASSWORD:-}
+      - BASIC_AUTH_PROXY_TIMEOUT=${AIRBYTE_BASIC_AUTH_PROXY_TIMEOUT:-}
+    networks:
+      - shared_data_network
+    depends_on:
+      - webapp
+      - server
+      - airbyte-api-server
 
 networks:
   shared_data_network: