SRIPHAT/sriphat-dataplatform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add airbyte docker compose

Browse Source
This commit is contained in:
jigoong
2026-02-17 09:00:21 +07:00
parent dbc52384ba
commit cdd27e1f5e
5 changed files with 196 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
04-ingestion/README.md
View File

@@ -1,10 +1,14 @@
# 04-ingestion: Airbyte Data Ingestion # 04-ingestion: Airbyte Data Ingestion
Airbyte OSS for data ingestion and ETL (standalone deployment). Airbyte OSS for data ingestion and ETL (multi-container deployment).
## Services ## Services
- **airbyte**: All-in-one Airbyte container (webapp + server + worker + temporal) - **airbyte-proxy**: Public entrypoint (UI/API gateway)
- **server**: Airbyte backend
- **worker**: Runs sync jobs and launches connector containers
- **webapp**: Airbyte UI
- **airbyte-temporal**: Workflow engine
## Run ## Run
@@ -15,14 +19,11 @@ docker compose --env-file ../.env.global up -d
## Access ## Access
- Web UI: http://localhost:8000 - Web UI: http://localhost:8000
- Configure in Nginx to route domain to `airbyte:8000` - Configure in Nginx to route domain to `airbyte-proxy:8000`
## Note ## Note
This uses Airbyte standalone image for simplicity. For production with high workload: This deployment pins Airbyte images to avoid `:latest` tag issues.
- Consider using full Airbyte stack with separate containers
- Or use `abctl` (Airbyte Command Line Tool)
- See: https://docs.airbyte.com/deploying-airbyte/docker-compose
## First Time Setup ## First Time Setup
1. Create database: `docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"` 1. Create database: `docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"`

201
04-ingestion/docker-compose.yml
View File

@@ -1,27 +1,192 @@
services: # services:
# Airbyte OSS - Simplified deployment # # Airbyte OSS - Simplified deployment
# Note: For production, consider using abctl or full Airbyte stack # # Note: For production, consider using abctl or full Airbyte stack
# This is a minimal setup for development/testing # # This is a minimal setup for development/testing
airbyte: # airbyte:
image: airbyte/airbyte-standalone:0.50.33 # image: airbyte/airbyte-standalone:0.50.33
container_name: airbyte # container_name: airbyte
ports: # ports:
- "8000:8000" # - "8000:8000"
environment: # environment:
- DATABASE_HOST=postgres # - DATABASE_HOST=postgres
- DATABASE_PORT=5432 # - DATABASE_PORT=5432
- DATABASE_USER=${DB_USER} # - DATABASE_USER=${DB_USER}
- DATABASE_PASSWORD=${DB_PASSWORD} # - DATABASE_PASSWORD=${DB_PASSWORD}
- DATABASE_DB=airbyte # - DATABASE_DB=airbyte
- TZ=${TZ:-Asia/Bangkok} # - TZ=${TZ:-Asia/Bangkok}
# volumes:
# - ./data/workspace:/workspace
# - ./data/config:/config
# - /var/run/docker.sock:/var/run/docker.sock
# networks:
# - shared_data_network
# restart: unless-stopped
# networks:
# shared_data_network:
# external: true
services:
docker-proxy:
image: alpine/socat
container_name: airbyte-docker-proxy
command: -t 900 TCP-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock
restart: unless-stopped
user: root
volumes: volumes:
- ./data/workspace:/workspace
- ./data/config:/config
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
networks: networks:
- shared_data_network - shared_data_network
airbyte-temporal:
image: airbyte/temporal:0.63.8
container_name: airbyte-temporal
restart: unless-stopped restart: unless-stopped
environment:
- DB=postgresql
- DB_PORT=${DB_PORT}
- POSTGRES_DB=temporal
- VISIBILITY_POSTGRES_DB=temporal_visibility
- POSTGRES_SEEDS=${DB_HOST}
- POSTGRES_USER=${DB_USER}
- POSTGRES_PWD=${DB_PASSWORD}
networks:
- shared_data_network
bootloader:
image: airbyte/bootloader:0.63.8
container_name: airbyte-bootloader
restart: "no"
environment:
- AIRBYTE_VERSION=0.63.8
- DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
- DATABASE_USER=${DB_USER}
- DATABASE_PASSWORD=${DB_PASSWORD}
- LOG_LEVEL=INFO
networks:
- shared_data_network
worker:
image: airbyte/worker:0.63.8
container_name: airbyte-worker
restart: unless-stopped
depends_on:
- docker-proxy
- airbyte-temporal
- bootloader
environment:
- AIRBYTE_VERSION=0.63.8
- DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
- DATABASE_USER=${DB_USER}
- DATABASE_PASSWORD=${DB_PASSWORD}
- CONFIG_ROOT=/data
- WORKSPACE_ROOT=/workspace
- LOCAL_ROOT=/local_root
- TEMPORAL_HOST=airbyte-temporal:7233
- LOG_LEVEL=INFO
- WORKER_ENVIRONMENT=docker
- DOCKER_HOST=docker-proxy:2375
volumes:
- ./data/workspace:/workspace
- ./data/config:/data
- ./data/local_root:/local_root
networks:
- shared_data_network
server:
image: airbyte/server:0.63.8
container_name: airbyte-server
restart: unless-stopped
depends_on:
- airbyte-temporal
- bootloader
environment:
- AIRBYTE_VERSION=0.63.8
- DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
- DATABASE_USER=${DB_USER}
- DATABASE_PASSWORD=${DB_PASSWORD}
- CONFIG_ROOT=/data
- WORKSPACE_ROOT=/workspace
- LOCAL_ROOT=/local_root
- TEMPORAL_HOST=airbyte-temporal:7233
- WEBAPP_URL=http://localhost:8000
- LOG_LEVEL=INFO
- WORKER_ENVIRONMENT=docker
volumes:
- ./data/workspace:/workspace
- ./data/config:/data
- ./data/local_root:/local_root
networks:
- shared_data_network
webapp:
image: airbyte/webapp:0.63.8
container_name: airbyte-webapp
restart: unless-stopped
environment:
- AIRBYTE_SERVER_HOST=server:8001
networks:
- shared_data_network
depends_on:
- server
airbyte-cron:
image: airbyte/cron:0.63.8
container_name: airbyte-cron
restart: unless-stopped
depends_on:
- bootloader
environment:
- AIRBYTE_VERSION=0.63.8
- DATABASE_URL=jdbc:postgresql://${DB_HOST}:${DB_PORT}/airbyte
- DATABASE_USER=${DB_USER}
- DATABASE_PASSWORD=${DB_PASSWORD}
- WORKSPACE_ROOT=/workspace
- TEMPORAL_HOST=airbyte-temporal:7233
- LOG_LEVEL=INFO
volumes:
- ./data/workspace:/workspace
networks:
- shared_data_network
airbyte-api-server:
image: airbyte/airbyte-api-server:0.63.8
container_name: airbyte-api-server
restart: unless-stopped
depends_on:
- bootloader
networks:
- shared_data_network
airbyte-connector-builder-server:
image: airbyte/connector-builder-server:0.63.8
container_name: airbyte-connector-builder-server
restart: unless-stopped
depends_on:
- bootloader
networks:
- shared_data_network
airbyte-proxy:
image: airbyte/proxy:0.63.8
container_name: airbyte-proxy
restart: unless-stopped
ports:
- "8000:8000"
- "8001:8001"
- "8003:8003"
- "8006:8006"
environment:
- BASIC_AUTH_USERNAME=${AIRBYTE_BASIC_AUTH_USERNAME:-}
- BASIC_AUTH_PASSWORD=${AIRBYTE_BASIC_AUTH_PASSWORD:-}
- BASIC_AUTH_PROXY_TIMEOUT=${AIRBYTE_BASIC_AUTH_PROXY_TIMEOUT:-}
networks:
- shared_data_network
depends_on:
- webapp
- server
- airbyte-api-server
networks: networks:
shared_data_network: shared_data_network:

2
DEPLOYMENT.md
View File

@@ -111,7 +111,7 @@ You should see:
**Airbyte:** **Airbyte:**
- Domain: `etl.sriphat.local` - Domain: `etl.sriphat.local`
- Forward Hostname: `airbyte` - Forward Hostname: `airbyte-proxy`
- Forward Port: `8000` - Forward Port: `8000`
### 2. Setup Keycloak SSO ### 2. Setup Keycloak SSO

2
setup-ubuntu.sh
View File

@@ -56,7 +56,7 @@ fi
# Install Docker Compose (standalone - backup) # Install Docker Compose (standalone - backup)
echo "[3/6] Installing Docker Compose standalone..." echo "[3/6] Installing Docker Compose standalone..."
if ! command -v docker-compose &> /dev/null; then if ! docker compose version &> /dev/null && ! command -v docker-compose &> /dev/null; then
DOCKER_COMPOSE_VERSION=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep 'tag_name' | cut -d\" -f4) DOCKER_COMPOSE_VERSION=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep 'tag_name' | cut -d\" -f4)
curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose

4
start-all.sh
View File

@@ -22,6 +22,8 @@ sleep 10
echo "[3/7] Creating databases for Airbyte and Superset..." echo "[3/7] Creating databases for Airbyte and Superset..."
docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'airbyte'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;" docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'airbyte'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"
docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'temporal'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE temporal;"
docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'temporal_visibility'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE temporal_visibility;"
docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'superset'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE superset;" docker exec postgres psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'superset'" | grep -q 1 || docker exec postgres psql -U postgres -c "CREATE DATABASE superset;"
echo "[4/7] Starting API Service..." echo "[4/7] Starting API Service..."
@@ -50,7 +52,7 @@ echo "- API Service: Configure in Nginx (apiservice:8000)"
echo "- Superset: Configure in Nginx (superset:8088)" echo "- Superset: Configure in Nginx (superset:8088)"
echo "" echo ""
echo "Next steps:" echo "Next steps:"
echo "1. Configure domains in Nginx Proxy Manager (port 81)" echo "1. Configure domains in Nginx Proxy Manager (port 8021)"
echo "2. Setup Keycloak realm and clients" echo "2. Setup Keycloak realm and clients"
echo "3. Configure Airbyte sources/destinations" echo "3. Configure Airbyte sources/destinations"
echo "4. Setup Superset dashboards" echo "4. Setup Superset dashboards"