add files infra docker service for data platform

04-ingestion/README.md

@@ -0,0 +1,19 @@
+# 04-ingestion: Airbyte Data Ingestion
+
+## Services
+- Airbyte Webapp
+- Airbyte Server
+- Airbyte Worker
+- Temporal (workflow engine)
+
+## Start
+```bash
+docker compose --env-file ../.env.global up -d
+```
+
+## Access
+Internal - configure Nginx Proxy Manager to expose at `/airbyte`
+
+## First Time Setup
+1. Create database: `docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"`
+2. Access webapp and configure sources/destinations

04-ingestion/docker-compose.yml

@@ -0,0 +1,70 @@
+services:
+  airbyte-webapp:
+    image: airbyte/webapp:latest
+    container_name: airbyte-webapp
+    environment:
+      - AIRBYTE_VERSION=latest
+      - API_URL=/api/v1/
+      - TRACKING_STRATEGY=segment
+    networks:
+      - shared_data_network
+    restart: unless-stopped
+    depends_on:
+      - airbyte-server
+
+  airbyte-server:
+    image: airbyte/server:latest
+    container_name: airbyte-server
+    environment:
+      - DATABASE_HOST=postgres
+      - DATABASE_PORT=5432
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - DATABASE_DB=airbyte
+      - CONFIG_DATABASE_USER=${DB_USER}
+      - CONFIG_DATABASE_PASSWORD=${DB_PASSWORD}
+      - WORKSPACE_ROOT=/tmp/workspace
+      - TRACKING_STRATEGY=segment
+      - TZ=${TZ:-Asia/Bangkok}
+    volumes:
+      - ./data/workspace:/tmp/workspace
+      - ./data/airbyte:/data
+    networks:
+      - shared_data_network
+    restart: unless-stopped
+
+  airbyte-worker:
+    image: airbyte/worker:latest
+    container_name: airbyte-worker
+    environment:
+      - DATABASE_HOST=postgres
+      - DATABASE_PORT=5432
+      - DATABASE_USER=${DB_USER}
+      - DATABASE_PASSWORD=${DB_PASSWORD}
+      - DATABASE_DB=airbyte
+      - WORKSPACE_ROOT=/tmp/workspace
+      - TZ=${TZ:-Asia/Bangkok}
+    volumes:
+      - ./data/workspace:/tmp/workspace
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - shared_data_network
+    restart: unless-stopped
+
+  airbyte-temporal:
+    image: temporalio/auto-setup:1.20.0
+    container_name: airbyte-temporal
+    environment:
+      - DB=postgresql
+      - DB_PORT=5432
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_PWD=${DB_PASSWORD}
+      - POSTGRES_SEEDS=postgres
+      - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml
+    networks:
+      - shared_data_network
+    restart: unless-stopped
+
+networks:
+  shared_data_network:
+    external: true