add airbyte

04-ingestion/README.md

@@ -1,18 +1,28 @@
 # 04-ingestion: Airbyte Data Ingestion
 
-## Services
-- Airbyte Webapp
-- Airbyte Server
-- Airbyte Worker
-- Temporal (workflow engine)
+Airbyte OSS for data ingestion and ETL (standalone deployment).
+
+## Services
+
+- **airbyte**: All-in-one Airbyte container (webapp + server + worker + temporal)
+
+## Run
 
-## Start
 ```bash
 docker compose --env-file ../.env.global up -d
 ```
 
 ## Access
-Internal - configure Nginx Proxy Manager to expose at `/airbyte`
+
+- Web UI: http://localhost:8000
+- Configure in Nginx to route domain to `airbyte:8000`
+
+## Note
+
+This uses Airbyte standalone image for simplicity. For production with high workload:
+- Consider using full Airbyte stack with separate containers
+- Or use `abctl` (Airbyte Command Line Tool)
+- See: https://docs.airbyte.com/deploying-airbyte/docker-compose
 
 ## First Time Setup
 1. Create database: `docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"`

04-ingestion/docker-compose.yml

@@ -1,70 +1,28 @@
 services:
-  airbyte-webapp:
-    image: airbyte/webapp:latest
-    container_name: airbyte-webapp
-    environment:
-      - AIRBYTE_VERSION=latest
-      - API_URL=/api/v1/
-      - TRACKING_STRATEGY=segment
-    networks:
-      - shared_data_network
-    restart: unless-stopped
-    depends_on:
-      - airbyte-server
-
-  airbyte-server:
-    image: airbyte/server:latest
-    container_name: airbyte-server
+  # Airbyte OSS - Simplified deployment
+  # Note: For production, consider using abctl or full Airbyte stack
+  # This is a minimal setup for development/testing
+  
+  airbyte:
+    image: airbyte/airbyte-standalone:0.50.33
+    container_name: airbyte
+    ports:
+      - "8000:8000"
     environment:
       - DATABASE_HOST=postgres
       - DATABASE_PORT=5432
       - DATABASE_USER=${DB_USER}
       - DATABASE_PASSWORD=${DB_PASSWORD}
       - DATABASE_DB=airbyte
-      - CONFIG_DATABASE_USER=${DB_USER}
-      - CONFIG_DATABASE_PASSWORD=${DB_PASSWORD}
-      - WORKSPACE_ROOT=/tmp/workspace
-      - TRACKING_STRATEGY=segment
       - TZ=${TZ:-Asia/Bangkok}
     volumes:
-      - ./data/workspace:/tmp/workspace
-      - ./data/airbyte:/data
-    networks:
-      - shared_data_network
-    restart: unless-stopped
-
-  airbyte-worker:
-    image: airbyte/worker:latest
-    container_name: airbyte-worker
-    environment:
-      - DATABASE_HOST=postgres
-      - DATABASE_PORT=5432
-      - DATABASE_USER=${DB_USER}
-      - DATABASE_PASSWORD=${DB_PASSWORD}
-      - DATABASE_DB=airbyte
-      - WORKSPACE_ROOT=/tmp/workspace
-      - TZ=${TZ:-Asia/Bangkok}
-    volumes:
-      - ./data/workspace:/tmp/workspace
+      - ./data/workspace:/workspace
+      - ./data/config:/config
       - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - shared_data_network
     restart: unless-stopped
 
-  airbyte-temporal:
-    image: temporalio/auto-setup:1.20.0
-    container_name: airbyte-temporal
-    environment:
-      - DB=postgresql
-      - DB_PORT=5432
-      - POSTGRES_USER=${DB_USER}
-      - POSTGRES_PWD=${DB_PASSWORD}
-      - POSTGRES_SEEDS=postgres
-      - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml
-    networks:
-      - shared_data_network
-    restart: unless-stopped
-
 networks:
   shared_data_network:
     external: true