diff --git a/04-ingestion/README.md b/04-ingestion/README.md index eccbbb6..50ca5bb 100644 --- a/04-ingestion/README.md +++ b/04-ingestion/README.md @@ -1,18 +1,28 @@ # 04-ingestion: Airbyte Data Ingestion -## Services -- Airbyte Webapp -- Airbyte Server -- Airbyte Worker -- Temporal (workflow engine) +Airbyte OSS for data ingestion and ETL (standalone deployment). + +## Services + +- **airbyte**: All-in-one Airbyte container (webapp + server + worker + temporal) + +## Run -## Start ```bash docker compose --env-file ../.env.global up -d ``` ## Access -Internal - configure Nginx Proxy Manager to expose at `/airbyte` + +- Web UI: http://localhost:8000 +- Configure in Nginx to route domain to `airbyte:8000` + +## Note + +This uses Airbyte standalone image for simplicity. For production with high workload: +- Consider using full Airbyte stack with separate containers +- Or use `abctl` (Airbyte Command Line Tool) +- See: https://docs.airbyte.com/deploying-airbyte/docker-compose ## First Time Setup 1. Create database: `docker exec postgres psql -U postgres -c "CREATE DATABASE airbyte;"` diff --git a/04-ingestion/docker-compose.yml b/04-ingestion/docker-compose.yml index 0695b13..5594093 100644 --- a/04-ingestion/docker-compose.yml +++ b/04-ingestion/docker-compose.yml @@ -1,70 +1,28 @@ services: - airbyte-webapp: - image: airbyte/webapp:latest - container_name: airbyte-webapp - environment: - - AIRBYTE_VERSION=latest - - API_URL=/api/v1/ - - TRACKING_STRATEGY=segment - networks: - - shared_data_network - restart: unless-stopped - depends_on: - - airbyte-server - - airbyte-server: - image: airbyte/server:latest - container_name: airbyte-server + # Airbyte OSS - Simplified deployment + # Note: For production, consider using abctl or full Airbyte stack + # This is a minimal setup for development/testing + + airbyte: + image: airbyte/airbyte-standalone:0.50.33 + container_name: airbyte + ports: + - "8000:8000" environment: - DATABASE_HOST=postgres - DATABASE_PORT=5432 - DATABASE_USER=${DB_USER} - DATABASE_PASSWORD=${DB_PASSWORD} - DATABASE_DB=airbyte - - CONFIG_DATABASE_USER=${DB_USER} - - CONFIG_DATABASE_PASSWORD=${DB_PASSWORD} - - WORKSPACE_ROOT=/tmp/workspace - - TRACKING_STRATEGY=segment - TZ=${TZ:-Asia/Bangkok} volumes: - - ./data/workspace:/tmp/workspace - - ./data/airbyte:/data - networks: - - shared_data_network - restart: unless-stopped - - airbyte-worker: - image: airbyte/worker:latest - container_name: airbyte-worker - environment: - - DATABASE_HOST=postgres - - DATABASE_PORT=5432 - - DATABASE_USER=${DB_USER} - - DATABASE_PASSWORD=${DB_PASSWORD} - - DATABASE_DB=airbyte - - WORKSPACE_ROOT=/tmp/workspace - - TZ=${TZ:-Asia/Bangkok} - volumes: - - ./data/workspace:/tmp/workspace + - ./data/workspace:/workspace + - ./data/config:/config - /var/run/docker.sock:/var/run/docker.sock networks: - shared_data_network restart: unless-stopped - airbyte-temporal: - image: temporalio/auto-setup:1.20.0 - container_name: airbyte-temporal - environment: - - DB=postgresql - - DB_PORT=5432 - - POSTGRES_USER=${DB_USER} - - POSTGRES_PWD=${DB_PASSWORD} - - POSTGRES_SEEDS=postgres - - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml - networks: - - shared_data_network - restart: unless-stopped - networks: shared_data_network: external: true diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 4de20f6..fc7c00d 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -76,17 +76,17 @@ You should see: | Service | URL | Default Credentials | |---------|-----|---------------------| -| **Nginx Proxy Manager** | http://localhost:81 | admin@example.com / changeme | +| **Nginx Proxy Manager** | http://localhost:8021 | admin@example.com / changeme | | **Keycloak Admin** | http://localhost:8080 | See KEYCLOAK_ADMIN in .env.global | -| **API Service** | http://localhost/apiservice | See ADMIN_USERNAME in .env.global | -| **Airbyte** | http://localhost/airbyte | Configure via Nginx first | -| **Superset** | http://localhost/superset | See SUPERSET_ADMIN_USERNAME in .env.global | +| **Airbyte** | http://localhost:8000 | No auth (setup via Nginx) | +| **API Service** | Configure via Nginx | See ADMIN_USERNAME in .env.global | +| **Superset** | Configure via Nginx | See SUPERSET_ADMIN_USERNAME in .env.global | ## 📝 Post-Installation Setup ### 1. Configure Nginx Proxy Manager -1. Access http://localhost:81 +1. Access http://localhost:8021 2. Login with default credentials (change on first login) 3. Add Proxy Hosts: @@ -111,7 +111,7 @@ You should see: **Airbyte:** - Domain: `etl.sriphat.local` -- Forward Hostname: `airbyte-webapp` +- Forward Hostname: `airbyte` - Forward Port: `8000` ### 2. Setup Keycloak SSO diff --git a/README-UBUNTU.md b/README-UBUNTU.md index aca32ed..28f0a05 100644 --- a/README-UBUNTU.md +++ b/README-UBUNTU.md @@ -100,8 +100,9 @@ docker logs apiservice hostname -I # Access points: -# - Nginx Proxy Manager: http://:81 +# - Nginx Proxy Manager: http://:8021 # - Keycloak: http://:8080 +# - Airbyte: http://:8000 ``` ### 3. Configure Firewall (if needed) diff --git a/install.sh b/install.sh index b9fe0eb..18ea337 100644 --- a/install.sh +++ b/install.sh @@ -102,8 +102,9 @@ echo "" echo "=== Installation Completed! ===" echo "" echo "Services are starting up. Wait 30-60 seconds, then access:" -echo "- Nginx Proxy Manager: http://$(hostname -I | awk '{print $1}'):81" +echo "- Nginx Proxy Manager: http://$(hostname -I | awk '{print $1}'):8021" echo "- Keycloak: http://$(hostname -I | awk '{print $1}'):8080" +echo "- Airbyte: http://$(hostname -I | awk '{print $1}'):8000" echo "" echo "Default credentials are in .env.global" echo "" diff --git a/setup-ubuntu.sh b/setup-ubuntu.sh index c604b49..aedbaf3 100644 --- a/setup-ubuntu.sh +++ b/setup-ubuntu.sh @@ -87,10 +87,11 @@ apt-get install -y \ echo "[6/6] Configuring firewall..." ufw --force enable ufw allow 22/tcp # SSH -ufw allow 80/tcp # HTTP -ufw allow 443/tcp # HTTPS -ufw allow 81/tcp # Nginx Proxy Manager Admin +ufw allow 8020/tcp # HTTP (Nginx) +ufw allow 8043/tcp # HTTPS (Nginx) +ufw allow 8021/tcp # Nginx Proxy Manager Admin ufw allow 8080/tcp # Keycloak (optional - can be removed after Nginx setup) +ufw allow 8000/tcp # Airbyte (optional - can be removed after Nginx setup) ufw status echo "" diff --git a/start-all.sh b/start-all.sh index 3c7fc51..d873a98 100644 --- a/start-all.sh +++ b/start-all.sh @@ -43,11 +43,11 @@ echo "" echo "=== All services started! ===" echo "" echo "Access points:" -echo "- Nginx Proxy Manager: http://localhost:81" +echo "- Nginx Proxy Manager: http://localhost:8021" echo "- Keycloak Admin: http://localhost:8080" -echo "- API Service: http://localhost/apiservice (via Nginx)" -echo "- Airbyte: http://localhost/airbyte (configure in Nginx)" -echo "- Superset: http://localhost/superset (configure in Nginx)" +echo "- Airbyte: http://localhost:8000" +echo "- API Service: Configure in Nginx (apiservice:8000)" +echo "- Superset: Configure in Nginx (superset:8088)" echo "" echo "Next steps:" echo "1. Configure domains in Nginx Proxy Manager (port 81)"