From 1dba772e620ab300dee9054b0e20406792d078f4 Mon Sep 17 00:00:00 2001 From: jigoong Date: Thu, 7 May 2026 17:57:42 +0700 Subject: [PATCH] update configuration docker setup for data platform --- 01-infra/docker-compose.yml | 99 ++- 01-infra/init/00-create-keycloak-database.sql | 5 + 01-infra/init/03-create-airflow-databases.sql | 8 + 01-infra/nginx-configs/README.md | 149 ++++ 01-infra/nginx-configs/airflow.conf | 67 ++ 01-infra/nginx-configs/apiservice.conf | 58 ++ 01-infra/nginx-configs/complete-example.conf | 144 ++++ 01-infra/nginx-configs/dbt.conf | 44 ++ 01-infra/nginx-configs/default-all.conf | 359 ++++++++++ 01-infra/nginx-configs/dozzle.conf | 59 ++ 01-infra/nginx-configs/keycloak.conf | 68 ++ .../nginx-proxy-manager-guide.md | 391 +++++++++++ 01-infra/nginx-configs/supabase-kong.conf | 88 +++ 01-infra/nginx-configs/supabase-studio.conf | 50 ++ 01-infra/nginx-configs/superset.conf | 68 ++ 03-apiservice/.env.example | 15 + 03-apiservice/.gitignore | 2 + 03-apiservice/AIRFLOW_INTEGRATION.md | 334 +++++++++ 03-apiservice/KEYCLOAK-SETUP.md | 325 +++++++++ 03-apiservice/README-PAGES.md | 226 +++++++ 03-apiservice/app/api/v1/routes.py | 88 ++- 03-apiservice/app/api/v1/schemas.py | 12 +- 03-apiservice/app/core/config.py | 15 + 03-apiservice/app/db/init_db.py | 44 +- 03-apiservice/app/db/models.py | 19 + 03-apiservice/app/db/session.py | 27 + 03-apiservice/app/main.py | 19 +- 03-apiservice/app/middleware/__init__.py | 1 + .../app/middleware/auth_middleware.py | 169 +++++ 03-apiservice/app/models/__init__.py | 7 + 03-apiservice/app/models/upload.py | 33 + 03-apiservice/app/models/user.py | 55 ++ 03-apiservice/app/routes/__init__.py | 1 + 03-apiservice/app/routes/admin_users.py | 156 +++++ 03-apiservice/app/routes/auth.py | 252 +++++++ 03-apiservice/app/routes/pages.py | 305 +++++++++ 03-apiservice/app/security/keycloak_auth.py | 146 ++++ 03-apiservice/app/security/permissions.py | 151 +++++ 03-apiservice/app/services/airflow_client.py | 152 +++++ 03-apiservice/app/templates/admin_users.html | 360 ++++++++++ .../templates/data_management_finance.html | 635 ++++++++++++++++++ 03-apiservice/app/templates/index.html | 296 ++++++++ 03-apiservice/app/utils/supabase_client.py | 4 +- 03-apiservice/docker-compose.yml | 19 +- 03-apiservice/requirements.txt | 3 + 06-analytics/Dockerfile.nginx | 46 ++ 06-analytics/nginx-superset.conf | 61 ++ 07-minio/.env.example | 55 ++ 07-minio/.gitignore | 30 + 07-minio/KEYCLOAK_INTEGRATION.md | 362 ++++++++++ 07-minio/README.md | 520 ++++++++++++++ 07-minio/docker-compose.yml | 50 ++ 07-minio/nginx-minio.conf | 104 +++ 53 files changed, 6732 insertions(+), 24 deletions(-) create mode 100644 01-infra/init/00-create-keycloak-database.sql create mode 100644 01-infra/init/03-create-airflow-databases.sql create mode 100644 01-infra/nginx-configs/README.md create mode 100644 01-infra/nginx-configs/airflow.conf create mode 100644 01-infra/nginx-configs/apiservice.conf create mode 100644 01-infra/nginx-configs/complete-example.conf create mode 100644 01-infra/nginx-configs/dbt.conf create mode 100644 01-infra/nginx-configs/default-all.conf create mode 100644 01-infra/nginx-configs/dozzle.conf create mode 100644 01-infra/nginx-configs/keycloak.conf create mode 100644 01-infra/nginx-configs/nginx-proxy-manager-guide.md create mode 100644 01-infra/nginx-configs/supabase-kong.conf create mode 100644 01-infra/nginx-configs/supabase-studio.conf create mode 100644 01-infra/nginx-configs/superset.conf create mode 100644 03-apiservice/AIRFLOW_INTEGRATION.md create mode 100644 03-apiservice/KEYCLOAK-SETUP.md create mode 100644 03-apiservice/README-PAGES.md create mode 100644 03-apiservice/app/db/session.py create mode 100644 03-apiservice/app/middleware/__init__.py create mode 100644 03-apiservice/app/middleware/auth_middleware.py create mode 100644 03-apiservice/app/models/__init__.py create mode 100644 03-apiservice/app/models/upload.py create mode 100644 03-apiservice/app/models/user.py create mode 100644 03-apiservice/app/routes/__init__.py create mode 100644 03-apiservice/app/routes/admin_users.py create mode 100644 03-apiservice/app/routes/auth.py create mode 100644 03-apiservice/app/routes/pages.py create mode 100644 03-apiservice/app/security/keycloak_auth.py create mode 100644 03-apiservice/app/security/permissions.py create mode 100644 03-apiservice/app/services/airflow_client.py create mode 100644 03-apiservice/app/templates/admin_users.html create mode 100644 03-apiservice/app/templates/data_management_finance.html create mode 100644 03-apiservice/app/templates/index.html create mode 100644 06-analytics/Dockerfile.nginx create mode 100644 06-analytics/nginx-superset.conf create mode 100644 07-minio/.env.example create mode 100644 07-minio/.gitignore create mode 100644 07-minio/KEYCLOAK_INTEGRATION.md create mode 100644 07-minio/README.md create mode 100644 07-minio/docker-compose.yml create mode 100644 07-minio/nginx-minio.conf diff --git a/01-infra/docker-compose.yml b/01-infra/docker-compose.yml index 97b1ad5..7b587e9 100644 --- a/01-infra/docker-compose.yml +++ b/01-infra/docker-compose.yml @@ -1,51 +1,76 @@ +x-common-configs: &common-config + extra_hosts: + - "dev.sriphat.com:192.168.100.9" + pull_policy: ${DOCKER_PULL_POLICY:-missing} + services: + # nginx-proxy: + # image: jc21/nginx-proxy-manager:latest + # container_name: nginx-proxy-manager + # ports: + # - "8020:80" + # - "8043:443" + # - "8021:81" + # volumes: + # - ./data:/data + # - ./letsencrypt:/etc/letsencrypt + # environment: + # - TZ=${TZ:-Asia/Bangkok} + # env_file: + # - ../.env + # networks: + # - shared_data_network + # restart: unless-stopped + nginx-proxy: - image: jc21/nginx-proxy-manager:latest + image: nginx:latest container_name: nginx-proxy-manager ports: - "8020:80" - - "8043:443" - - "8021:81" - volumes: - - ./data:/data - - ./letsencrypt:/etc/letsencrypt environment: - TZ=${TZ:-Asia/Bangkok} - env_file: - - ../.env.global + volumes: + - ./nginx-configs/default-all.conf:/etc/nginx/conf.d/default.conf:ro networks: - shared_data_network restart: unless-stopped + <<: *common-config keycloak: image: quay.io/keycloak/keycloak:23.0 container_name: keycloak - command: start-dev + #command: start-dev + command: start-dev --http-relative-path /keycloak env_file: - - ../.env.global + - ../.env environment: KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN} KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} KC_DB: postgres - KC_DB_URL: jdbc:postgresql://postgres:5432/${KEYCLOAK_DB_NAME} + KC_DB_URL: jdbc:postgresql://postgres:${DB_PORT:-5432}/${KEYCLOAK_DB_NAME} KC_DB_USERNAME: ${DB_USER} KC_DB_PASSWORD: ${DB_PASSWORD} KC_HOSTNAME_STRICT: "false" KC_HTTP_ENABLED: "true" KC_PROXY: edge + # passthrough + KC_HTTP_RELATIVE_PATH: "/keycloak" + KC_HOSTNAME_PATH: "/keycloak" + KC_HOSTNAME_STRICT_HTTPS: "true" ports: - - "8080:8080" + - "8085:8080" networks: - shared_data_network restart: unless-stopped depends_on: - postgres - + <<: *common-config + postgres: image: postgres:15-alpine container_name: postgres env_file: - - ../.env.global + - ../.env environment: POSTGRES_PASSWORD: ${DB_PASSWORD} POSTGRES_USER: ${DB_USER} @@ -58,13 +83,57 @@ services: - shared_data_network restart: unless-stopped ports: - - "0.0.0.0:5435:5432" + - "0.0.0.0:${DB_PORT_EXPOSE:-5435}:5432" healthcheck: test: ["CMD-SHELL", "pg_isready -U ${DB_USER}"] interval: 10s timeout: 5s retries: 5 + redis: + # Redis is limited to 7.2-bookworm due to licencing change + # https://redis.io/blog/redis-adopts-dual-source-available-licensing/ + image: redis:7.2-bookworm + expose: + - 6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 30s + retries: 50 + start_period: 30s + restart: always + networks: + - shared_data_network + + dozzle: + image: amir20/dozzle:latest + container_name: dozzle + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./data/dozzle:/data + ports: + - "${DOZZLE_PORT:-9999}:8080" + environment: + DOZZLE_LEVEL: ${DOZZLE_LEVEL:-info} + DOZZLE_BASE: ${DOZZLE_BASE:-/dozzle} + DOZZLE_HOSTNAME: ${DOZZLE_HOSTNAME:-Sriphat Main Server} + DOZZLE_NO_ANALYTICS: "true" + DOZZLE_ENABLE_ACTIONS: "true" + DOZZLE_AUTH_PROVIDER: ${DOZZLE_AUTH_PROVIDER:-none} + DOZZLE_REMOTE_AGENT: ${DOZZLE_REMOTE_AGENT:-} + TZ: ${TZ:-Asia/Bangkok} + networks: + - shared_data_network + restart: ${DOZZLE_RESTART_POLICY:-unless-stopped} + healthcheck: + test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/dozzle/healthcheck"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + <<: *common-config + networks: shared_data_network: external: true diff --git a/01-infra/init/00-create-keycloak-database.sql b/01-infra/init/00-create-keycloak-database.sql new file mode 100644 index 0000000..c7f4691 --- /dev/null +++ b/01-infra/init/00-create-keycloak-database.sql @@ -0,0 +1,5 @@ +-- Create Keycloak database +CREATE DATABASE keycloak; + +-- Grant privileges to postgres user +GRANT ALL PRIVILEGES ON DATABASE keycloak TO postgres; diff --git a/01-infra/init/03-create-airflow-databases.sql b/01-infra/init/03-create-airflow-databases.sql new file mode 100644 index 0000000..4d77425 --- /dev/null +++ b/01-infra/init/03-create-airflow-databases.sql @@ -0,0 +1,8 @@ +-- Create databases for Airbyte OSS +-- These databases will be used by the Airbyte deployment in 04-ingestion + +-- Main Airbyte database +CREATE DATABASE airflow_db; + +-- Grant permissions to postgres user +GRANT ALL PRIVILEGES ON DATABASE airflow_db TO postgres; diff --git a/01-infra/nginx-configs/README.md b/01-infra/nginx-configs/README.md new file mode 100644 index 0000000..23d62e6 --- /dev/null +++ b/01-infra/nginx-configs/README.md @@ -0,0 +1,149 @@ +# Nginx Proxy Manager - Site Configurations + +Nginx site configurations สำหรับ handle subpath routing ของทุก service ใน Sriphat Data Platform + +## 📋 Services และ Subpaths + +| Service | Subpath | Backend | Port | +|---------|---------|---------|------| +| API Service | `/apiservice` | apiservice:8000 | 8040 | +| Supabase Studio | `/supabase` | sdp-studio:3000 | - | +| Supabase Kong API | `/supabase-api` | sdp-kong:8000 | - | +| Keycloak | `/keycloak` | keycloak:8080 | 8085 | +| Superset | `/superset` | superset:8088 | 8088 | +| Airflow | `/airflow` | airflow-webserver:8080 | - | +| DBT Docs | `/dbt` | dbt-docs:8080 | - | +| Dozzle | `/dozzle` | dozzle:8080 | 9999 | + +## 🚀 การใช้งาน + +### วิธีที่ 1: ใช้ Nginx Proxy Manager UI (แนะนำ) + +1. เข้า Nginx Proxy Manager: `http://your-server:8021` +2. Login (default: admin@example.com / changeme) +3. ไปที่ **Proxy Hosts** → **Add Proxy Host** +4. กรอกข้อมูล: + - **Domain Names**: `ai.sriphat.com` (หรือ domain ของคุณ) + - **Scheme**: `http` + - **Forward Hostname/IP**: ดูจากตารางด้านบน + - **Forward Port**: ดูจากตารางด้านบน +5. ไปที่ tab **Advanced** +6. Copy config จากไฟล์ที่เกี่ยวข้อง (เช่น `apiservice.conf`) ไปวางใน **Custom Nginx Configuration** +7. กด **Save** + +### วิธีที่ 2: Import Config Files โดยตรง + +**⚠️ วิธีนี้ต้องการ access ไปยัง Nginx Proxy Manager data directory** + +```bash +# 1. Copy config files ไปยัง Nginx Proxy Manager +cd /path/to/01-infra +docker cp nginx-configs/. nginx-proxy-manager:/data/nginx/custom/ + +# 2. Restart Nginx Proxy Manager +docker restart nginx-proxy-manager + +# 3. ตรวจสอบ logs +docker logs nginx-proxy-manager -f +``` + +## 📝 Config Files + +### `apiservice.conf` +FastAPI service with Keycloak authentication +- Handles `/apiservice/*` paths +- Preserves session cookies +- WebSocket support + +### `supabase-studio.conf` +Supabase Studio UI +- Handles `/supabase/*` paths +- Rewrites paths for Studio + +### `supabase-kong.conf` +Supabase REST API (Kong Gateway) +- Handles `/supabase-api/*` paths +- API key authentication + +### `keycloak.conf` +Keycloak SSO +- Handles `/keycloak/*` paths +- Preserves authentication headers + +### `superset.conf` +Apache Superset BI +- Handles `/superset/*` paths +- Session management + +### `airflow.conf` +Apache Airflow (if deployed) +- Handles `/airflow/*` paths +- WebServer UI + +### `dbt.conf` +DBT Documentation (if deployed) +- Handles `/dbt/*` paths +- Static documentation + +## 🔧 การปรับแต่ง + +### เปลี่ยน Domain +แก้ไข `server_name` ในแต่ละ config file: +```nginx +server_name ai.sriphat.com; # เปลี่ยนเป็น domain ของคุณ +``` + +### เปลี่ยน Backend Host/Port +แก้ไข `proxy_pass` directive: +```nginx +proxy_pass http://apiservice:8000; # เปลี่ยนตาม service ของคุณ +``` + +### เพิ่ม SSL/HTTPS +ใช้ Nginx Proxy Manager UI: +1. ไปที่ Proxy Host ที่ต้องการ +2. ไปที่ tab **SSL** +3. เลือก **Request a new SSL Certificate** +4. เลือก **Force SSL** + +## 🐛 Troubleshooting + +### 502 Bad Gateway +- ตรวจสอบว่า backend service ทำงานอยู่: `docker ps` +- ตรวจสอบ network: `docker network inspect shared_data_network` +- ดู logs: `docker logs ` + +### 404 Not Found +- ตรวจสอบ path rewriting ใน config +- ดู nginx logs: `docker logs nginx-proxy-manager` + +### Session/Cookie Issues +- ตรวจสอบ `proxy_cookie_path` directive +- ตรวจสอบ `X-Forwarded-*` headers + +### WebSocket Connection Failed +- ตรวจสอบว่ามี WebSocket headers: + ```nginx + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + ``` + +## 📚 เอกสารเพิ่มเติม + +- [Nginx Proxy Manager Documentation](https://nginxproxymanager.com/guide/) +- [Nginx Reverse Proxy Guide](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) +- [Nginx Subpath Configuration](https://www.nginx.com/blog/creating-nginx-rewrite-rules/) + +## 🔐 Security Notes + +1. **ใช้ HTTPS ใน Production** - Request SSL certificate ผ่าน Nginx Proxy Manager +2. **ตั้งค่า Access Lists** - จำกัดการเข้าถึงบาง services (เช่น Keycloak Admin) +3. **Enable Rate Limiting** - ป้องกัน DDoS attacks +4. **Update Regularly** - อัพเดท Nginx Proxy Manager เป็นประจำ + +## 📞 Support + +หากมีปัญหาหรือข้อสงสัย: +1. ตรวจสอบ logs: `docker logs nginx-proxy-manager -f` +2. ดู Nginx Proxy Manager UI → **Logs** +3. ตรวจสอบ backend service logs diff --git a/01-infra/nginx-configs/airflow.conf b/01-infra/nginx-configs/airflow.conf new file mode 100644 index 0000000..8ccb858 --- /dev/null +++ b/01-infra/nginx-configs/airflow.conf @@ -0,0 +1,67 @@ +# Apache Airflow - Workflow Orchestration +# Subpath: /airflow +# Backend: airflow-webserver:8080 + +location /airflow { + # Remove /airflow prefix before forwarding + rewrite ^/airflow(/.*)$ $1 break; + + # Forward to Airflow Webserver + proxy_pass http://airflow-webserver:8080; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # Important for Airflow subpath + proxy_set_header X-Script-Name /airflow; + + # Session cookie handling + proxy_cookie_path / /airflow/; + + # WebSocket support for real-time logs + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Timeouts (DAG runs can take time) + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + # Buffer settings + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; +} + +# API endpoints +location /airflow/api { + rewrite ^/airflow(/.*)$ $1 break; + proxy_pass http://airflow-webserver:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Script-Name /airflow; + proxy_cookie_path / /airflow/; +} + +# Static files +location /airflow/static { + rewrite ^/airflow(/.*)$ $1 break; + proxy_pass http://airflow-webserver:8080; + proxy_set_header Host $host; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; +} + +# Health check +location /airflow/health { + rewrite ^/airflow(/.*)$ $1 break; + proxy_pass http://airflow-webserver:8080; + proxy_set_header Host $host; +} diff --git a/01-infra/nginx-configs/apiservice.conf b/01-infra/nginx-configs/apiservice.conf new file mode 100644 index 0000000..3ed95e6 --- /dev/null +++ b/01-infra/nginx-configs/apiservice.conf @@ -0,0 +1,58 @@ +# API Service - FastAPI with Keycloak Authentication +# Subpath: /apiservice +# Backend: apiservice:8000 + +location /apiservice { + # Remove /apiservice prefix before forwarding to backend + rewrite ^/apiservice(/.*)$ $1 break; + + # Forward to FastAPI backend + proxy_pass http://apiservice:8000; + + # Preserve original host and protocol + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + + # Important: Tell FastAPI about the subpath + proxy_set_header X-Script-Name /apiservice; + + # Session cookie handling + proxy_cookie_path / /apiservice/; + + # WebSocket support (for future use) + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Timeouts + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # Buffer settings + proxy_buffering off; + proxy_request_buffering off; +} + +# Static files (if any) +location /apiservice/static { + rewrite ^/apiservice/static(/.*)$ /static$1 break; + proxy_pass http://apiservice:8000; + proxy_set_header Host $host; +} + +# Admin panel +location /apiservice/admin { + rewrite ^/apiservice(/.*)$ $1 break; + proxy_pass http://apiservice:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Script-Name /apiservice; + proxy_cookie_path / /apiservice/; +} diff --git a/01-infra/nginx-configs/complete-example.conf b/01-infra/nginx-configs/complete-example.conf new file mode 100644 index 0000000..d1e7e27 --- /dev/null +++ b/01-infra/nginx-configs/complete-example.conf @@ -0,0 +1,144 @@ +# Complete Nginx Configuration Example +# สำหรับ Nginx Proxy Manager - Custom Nginx Configuration +# +# วิธีใช้: +# 1. ไปที่ Nginx Proxy Manager UI (http://your-server:8021) +# 2. สร้าง Proxy Host ใหม่ +# 3. กรอก Domain Names: ai.sriphat.com (หรือ domain ของคุณ) +# 4. กรอก Forward Hostname/IP: localhost (dummy, จะใช้ config ด้านล่าง) +# 5. กรอก Forward Port: 80 (dummy) +# 6. ไปที่ tab "Advanced" +# 7. Copy config ด้านล่างนี้ทั้งหมดไปวางใน "Custom Nginx Configuration" +# 8. กด Save + +# ============================================================================ +# API Service - FastAPI with Keycloak +# ============================================================================ +location /apiservice { + rewrite ^/apiservice(/.*)$ $1 break; + proxy_pass http://apiservice:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Script-Name /apiservice; + proxy_cookie_path / /apiservice/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; +} + +# ============================================================================ +# Supabase Studio - Database Management UI +# ============================================================================ +location /supabase { + rewrite ^/supabase(/.*)$ $1 break; + proxy_pass http://sdp-studio:3000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; +} + +# ============================================================================ +# Supabase Kong API - REST API Gateway +# ============================================================================ +location /supabase-api { + rewrite ^/supabase-api(/.*)$ $1 break; + proxy_pass http://sdp-kong:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Authorization $http_authorization; + proxy_set_header apikey $http_apikey; + proxy_buffering off; +} + +# ============================================================================ +# Keycloak - Single Sign-On (SSO) +# ============================================================================ +location /keycloak { + rewrite ^/keycloak(/.*)$ $1 break; + proxy_pass http://keycloak:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Prefix /keycloak; + proxy_cookie_path / /keycloak/; + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; +} + +# ============================================================================ +# Apache Superset - Business Intelligence +# ============================================================================ +location /superset { + rewrite ^/superset(/.*)$ $1 break; + proxy_pass http://superset:8088; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Script-Name /superset; + proxy_cookie_path / /superset/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; +} + +# ============================================================================ +# Apache Airflow - Workflow Orchestration (Optional) +# ============================================================================ +location /airflow { + rewrite ^/airflow(/.*)$ $1 break; + proxy_pass http://airflow-webserver:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Script-Name /airflow; + proxy_cookie_path / /airflow/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; +} + +# ============================================================================ +# DBT Documentation (Optional) +# ============================================================================ +location /dbt { + rewrite ^/dbt(/.*)$ $1 break; + proxy_pass http://dbt-docs:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +# ============================================================================ +# Dozzle - Docker Log Viewer & Monitoring +# ============================================================================ +location /dozzle { + proxy_pass http://dozzle:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; + proxy_read_timeout 300s; +} diff --git a/01-infra/nginx-configs/dbt.conf b/01-infra/nginx-configs/dbt.conf new file mode 100644 index 0000000..586b8ce --- /dev/null +++ b/01-infra/nginx-configs/dbt.conf @@ -0,0 +1,44 @@ +# DBT Documentation - Data Transformation Docs +# Subpath: /dbt +# Backend: dbt-docs:8080 + +location /dbt { + # Remove /dbt prefix before forwarding + rewrite ^/dbt(/.*)$ $1 break; + + # Forward to DBT docs server + proxy_pass http://dbt-docs:8080; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Timeouts + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; +} + +# Static documentation files +location /dbt/static { + rewrite ^/dbt(/.*)$ $1 break; + proxy_pass http://dbt-docs:8080; + proxy_set_header Host $host; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; +} + +# Catalog and manifest files +location /dbt/catalog.json { + rewrite ^/dbt(/.*)$ $1 break; + proxy_pass http://dbt-docs:8080; + proxy_set_header Host $host; +} + +location /dbt/manifest.json { + rewrite ^/dbt(/.*)$ $1 break; + proxy_pass http://dbt-docs:8080; + proxy_set_header Host $host; +} diff --git a/01-infra/nginx-configs/default-all.conf b/01-infra/nginx-configs/default-all.conf new file mode 100644 index 0000000..adc3083 --- /dev/null +++ b/01-infra/nginx-configs/default-all.conf @@ -0,0 +1,359 @@ +server { + listen 80; + server_name dev.sriphat.com; + + client_max_body_size 100M; + + # redirect to ai web while wait for main protal web in the future + location = / { + return 301 /ai/; + } + + + location /keycloak/ { + #rewrite ^/keycloak/(.*)$ /$1 break; + proxy_pass http://keycloak:8080; + + # Add WebSocket support (Necessary for version 0.5.0 and up) + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # (Optional) Disable proxy buffering for better streaming response from models + proxy_buffering off; + + # (Optional) Increase max request size for large attachments and long audio messages + #client_max_body_size 20M; + proxy_read_timeout 10m; + } + + + # location /supabase2/ { + # #rewrite ^/supabase2/(.*)$ /$1 break; + # proxy_pass http://sdp-kong:8000; + + # # Add WebSocket support (Necessary for version 0.5.0 and up) + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection "upgrade"; + + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + + # # (Optional) Disable proxy buffering for better streaming response from models + # proxy_buffering off; + + # # (Optional) Increase max request size for large attachments and long audio messages + # #client_max_body_size 20M; + # proxy_read_timeout 10m; + # } + + + # location /ai/ { + # proxy_pass http://localhost:3001/ai/; + + # # Add WebSocket support (Necessary for version 0.5.0 and up) + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection "upgrade"; + + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + + # # (Optional) Disable proxy buffering for better streaming response from models + # proxy_buffering off; + + # # (Optional) Increase max request size for large attachments and long audio messages + # #client_max_body_size 20M; + # proxy_read_timeout 10m; + # } + + # location /dashboard/ { + # proxy_pass http://localhost:8800; + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection 'upgrade'; + # proxy_set_header Host $host; + # proxy_cache_bypass $http_upgrade; + # } + + # location /dashboard-dev/ { + # proxy_pass http://localhost:8801; + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection 'upgrade'; + # proxy_set_header Host $host; + # proxy_cache_bypass $http_upgrade; + # } + + # location /realtime/ { + # proxy_pass http://sdp-kong:8000/realtime/; # ส่งไปที่ endpoint ของ backend supabase + + # # คอนฟิกสำหรับ WebSocket + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection "Upgrade"; + + # # Header สำคัญอื่นๆ + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # # ป้องกันแชทหลุด (Timeout 1 ชั่วโมง) + # proxy_read_timeout 3600s; + # proxy_send_timeout 3600s; + # } + + + location /apiservice/ { + # ส่งต่อ Request ไปยัง Backend + proxy_pass http://apiservice:8040; + + # การตั้งค่า Header มาตรฐาน + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # Ensure the sub-path is handled correctly + proxy_set_header X-Forwarded-Prefix /apiservice; + + # รองรับการ Upload ไฟล์ขนาดใหญ่ (ป้องกัน Timeout ระหว่างส่งข้อมูล) + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + + # ปิดการพักข้อมูลใน Buffer ของ Nginx ชั่วคราวเพื่อให้การ Upload ลื่นไหลขึ้น + proxy_request_buffering off; + proxy_buffering off; + + # เพิ่มเติม: รองรับ WebSocket (เผื่อ Doc หรือ API มีการใช้ Real-time) + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + + # # Superset Analytics Dashboard -- notwork + location /superset { + proxy_pass http://superset:8088; + + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Standard headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Superset-specific headers for sub-path support + proxy_set_header X-Forwarded-Prefix /superset; + proxy_set_header X-Script-Name /superset; + # ตัวนี้จะช่วยให้ Superset เข้าใจเรื่อง Path ในการสร้างลิงก์ Static + proxy_set_header X-Forwarded-Host $host; + + # Timeout settings (for long-running queries and dashboard loading) + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + + # Disable buffering for better streaming + proxy_buffering off; + proxy_request_buffering off; + } + + location /dozzle { + # Forward to Dozzle + proxy_pass http://dozzle:8080; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # WebSocket support for real-time logs + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Timeouts for long-running log streams + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + # Disable buffering for real-time streaming + proxy_buffering off; + proxy_request_buffering off; + } + + # Static assets + location /dozzle/assets { + proxy_pass http://dozzle:8080; + proxy_set_header Host $host; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; + } + + # API endpoints + location /dozzle/api { + proxy_pass http://dozzle:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; + } + + # Health check + location /dozzle/healthcheck { + proxy_pass http://dozzle:8080; + proxy_set_header Host $host; + access_log off; + } + + # ============================================= + # Superset Analytics Dashboard + # Superset routes are at root level (no single prefix) + # We proxy ALL Superset paths directly to port 8088 + # ============================================= + + # # Redirect /superset/ and /superset to welcome page + # location = /superset/ { + # return 302 /superset/welcome/; + # } + # location = /superset { + # return 302 /superset/welcome/; + # } + + # Superset views (welcome, dashboard view, explore, etc.) + # location /superset { + # proxy_pass http://localhost:8088; + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection 'upgrade'; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # proxy_set_header Accept-Encoding ""; + # proxy_read_timeout 300s; + # proxy_connect_timeout 300s; + # proxy_send_timeout 300s; + + # # Rewrite logo brand link in HTML + # sub_filter_once off; + # sub_filter '"path":"/"' '"path":"/superset/welcome/"'; + # sub_filter '"path": "/"' '"path": "/superset/welcome/"'; + # } + + # # Superset API, static files, auth, and all other root-level routes + # # NOTE: /dashboard/ is NOT included here - it's handled by sriphat-dashboard on port 8800 + # location ~ ^/(api|static|login|logout|chart|explore|sqllab|savedqueryview|tablemodelview|tableschemaview|tabstateview|tagview|datasource|dataset|databaseview|annotationlayer|csstemplatemodelview|rowlevelsecurity|embedded|dynamic-plugins|lang|theme|healthcheck|ping|roles|users|user_info|userinfoeditview|register|registrations|resetpassword|resetmypassword|groups|list_groups|back|swagger|alert|report|actionlog)(/|$) { + # proxy_pass http://localhost:8088; + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection 'upgrade'; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # proxy_set_header Accept-Encoding ""; + # proxy_read_timeout 300s; + # proxy_connect_timeout 300s; + # proxy_send_timeout 300s; + + # # Rewrite logo href in HTML/JS responses + # sub_filter_once off; + # sub_filter_types application/javascript; + # sub_filter '"path":"/"' '"path":"/superset/welcome/"'; + # sub_filter "'path':'/'" "'path':'/superset/welcome/'"; + # } + + # location /aiflow/ { + # proxy_pass http://airflow-webserver:8080; + + # # WebSocket support + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection "upgrade"; + + # # Standard headers + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + + # # Superset-specific headers for sub-path support + # proxy_set_header X-Forwarded-Prefix /superset; + # proxy_set_header X-Script-Name /superset; + + # # Timeout settings (for long-running queries and dashboard loading) + # proxy_read_timeout 300s; + # proxy_connect_timeout 300s; + # proxy_send_timeout 300s; + + # # Disable buffering for better streaming + # proxy_buffering off; + # proxy_request_buffering off; + # } + + # location /dbt/ { + # proxy_pass http://dbt:8080; + + # # WebSocket support + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection "upgrade"; + + # # Standard headers + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + + # # Superset-specific headers for sub-path support + # proxy_set_header X-Forwarded-Prefix /superset; + # proxy_set_header X-Script-Name /superset; + + # # Timeout settings (for long-running queries and dashboard loading) + # proxy_read_timeout 300s; + # proxy_connect_timeout 300s; + # proxy_send_timeout 300s; + + # # Disable buffering for better streaming + # proxy_buffering off; + # proxy_request_buffering off; + # } + + #listen 443 ssl; # managed by sriphat + #ssl_certificate /etc/letsencrypt/live/ai.bda.co.th/fullchain.pem; # managed by Certbot + #ssl_certificate_key /etc/letsencrypt/live/ai.bda.co.th/privkey.pem; # managed by Certbot + #include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + #ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} + +#server { +# listen 80 default_server; +# server_name ai.bda.co.th; +# #rewrite ^/[old-page]$ https://[domain]/[new-page] permanent; +# return 301 https://$host$request_uri; +#} \ No newline at end of file diff --git a/01-infra/nginx-configs/dozzle.conf b/01-infra/nginx-configs/dozzle.conf new file mode 100644 index 0000000..8cde93d --- /dev/null +++ b/01-infra/nginx-configs/dozzle.conf @@ -0,0 +1,59 @@ +# Dozzle - Docker Log Viewer & Monitoring +# Subpath: /dozzle +# Backend: dozzle:8080 + + location /dozzle { + # Forward to Dozzle + proxy_pass http://dozzle:8080; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # WebSocket support for real-time logs + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Timeouts for long-running log streams + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + # Disable buffering for real-time streaming + proxy_buffering off; + proxy_request_buffering off; + } + + # Static assets + location /dozzle/assets { + proxy_pass http://dozzle:8080; + proxy_set_header Host $host; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; + } + + # API endpoints + location /dozzle/api { + proxy_pass http://dozzle:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; + } + + # Health check + location /dozzle/healthcheck { + proxy_pass http://dozzle:8080; + proxy_set_header Host $host; + access_log off; + } diff --git a/01-infra/nginx-configs/keycloak.conf b/01-infra/nginx-configs/keycloak.conf new file mode 100644 index 0000000..76086da --- /dev/null +++ b/01-infra/nginx-configs/keycloak.conf @@ -0,0 +1,68 @@ +# Keycloak - Single Sign-On (SSO) +# Subpath: /keycloak +# Backend: keycloak:8080 + +location /keycloak { + # Remove /keycloak prefix before forwarding + rewrite ^/keycloak(/.*)$ $1 break; + + # Forward to Keycloak + proxy_pass http://keycloak:8080; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port $server_port; + + # Important for Keycloak + proxy_set_header X-Forwarded-Prefix /keycloak; + + # Session and cookie handling + proxy_cookie_path / /keycloak/; + + # Buffer settings + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; + + # Timeouts + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; +} + +# Admin console +location /keycloak/admin { + rewrite ^/keycloak(/.*)$ $1 break; + proxy_pass http://keycloak:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Prefix /keycloak; + proxy_cookie_path / /keycloak/; +} + +# Realms +location /keycloak/realms { + rewrite ^/keycloak(/.*)$ $1 break; + proxy_pass http://keycloak:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Prefix /keycloak; + proxy_cookie_path / /keycloak/; +} + +# Resources (CSS, JS, images) +location /keycloak/resources { + rewrite ^/keycloak(/.*)$ $1 break; + proxy_pass http://keycloak:8080; + proxy_set_header Host $host; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; +} diff --git a/01-infra/nginx-configs/nginx-proxy-manager-guide.md b/01-infra/nginx-configs/nginx-proxy-manager-guide.md new file mode 100644 index 0000000..c2d8fbf --- /dev/null +++ b/01-infra/nginx-configs/nginx-proxy-manager-guide.md @@ -0,0 +1,391 @@ +# Nginx Proxy Manager - Setup Guide + +คู่มือการตั้งค่า Nginx Proxy Manager สำหรับ Sriphat Data Platform + +## 📋 ข้อมูล Services + +| Service | Subpath | Container Name | Port | Status | +|---------|---------|----------------|------|--------| +| API Service | `/apiservice` | apiservice | 8000 | ✅ Active | +| Supabase Studio | `/supabase` | sdp-studio | 3000 | ✅ Active | +| Supabase API | `/supabase-api` | sdp-kong | 8000 | ✅ Active | +| Keycloak | `/keycloak` | keycloak | 8080 | ✅ Active | +| Superset | `/superset` | superset | 8088 | ✅ Active | +| Airflow | `/airflow` | airflow-webserver | 8080 | ⚠️ Optional | +| DBT Docs | `/dbt` | dbt-docs | 8080 | ⚠️ Optional | + +## 🚀 Quick Start + +### 1. เข้า Nginx Proxy Manager + +``` +URL: http://192.168.100.9:8021 +Default Login: + Email: admin@example.com + Password: changeme +``` + +**⚠️ เปลี่ยน password ทันทีหลัง login ครั้งแรก!** + +### 2. สร้าง Proxy Host + +1. คลิก **Proxy Hosts** → **Add Proxy Host** +2. กรอกข้อมูล: + +**Tab: Details** +``` +Domain Names: ai.sriphat.com +Scheme: http +Forward Hostname/IP: localhost +Forward Port: 80 +``` + +**Tab: Advanced** +- Copy config จาก `complete-example.conf` ทั้งหมด +- Paste ลงใน **Custom Nginx Configuration** + +3. กด **Save** + +### 3. ตั้งค่า SSL (Production) + +**Tab: SSL** +``` +☑ Request a new SSL Certificate with Let's Encrypt +☑ Force SSL +☑ HTTP/2 Support +☑ HSTS Enabled + +Email: your-email@example.com +``` + +กด **Save** + +## 📝 การใช้งานแบบละเอียด + +### วิธีที่ 1: ใช้ Complete Config (แนะนำ) + +**ข้อดี:** +- ตั้งค่าครั้งเดียว ได้ทุก service +- ง่ายต่อการจัดการ +- Consistent configuration + +**ขั้นตอน:** +1. สร้าง Proxy Host ตาม Quick Start +2. Copy `complete-example.conf` ไปวางใน Advanced tab +3. Save + +### วิธีที่ 2: แยก Config แต่ละ Service + +**ข้อดี:** +- ควบคุมแต่ละ service ได้อิสระ +- ง่ายต่อการ debug +- สามารถตั้งค่า SSL แยกกันได้ + +**ขั้นตอน:** + +#### API Service +``` +Domain: api.sriphat.com +Forward: apiservice:8000 +Advanced: ใช้ config จาก apiservice.conf +``` + +#### Supabase Studio +``` +Domain: supabase.sriphat.com +Forward: sdp-studio:3000 +Advanced: ใช้ config จาก supabase-studio.conf +``` + +#### Keycloak +``` +Domain: auth.sriphat.com +Forward: keycloak:8080 +Advanced: ใช้ config จาก keycloak.conf +``` + +#### Superset +``` +Domain: bi.sriphat.com +Forward: superset:8088 +Advanced: ใช้ config จาก superset.conf +``` + +## 🔧 Configuration Details + +### API Service (`/apiservice`) + +**สิ่งสำคัญ:** +```nginx +proxy_set_header X-Script-Name /apiservice; +proxy_cookie_path / /apiservice/; +``` + +**ทำไม:** +- FastAPI ต้องรู้ว่าทำงานภายใต้ subpath +- Session cookies ต้อง scope ถูกต้อง + +### Keycloak (`/keycloak`) + +**สิ่งสำคัญ:** +```nginx +proxy_set_header X-Forwarded-Prefix /keycloak; +proxy_cookie_path / /keycloak/; +``` + +**ทำไม:** +- Keycloak ใช้ X-Forwarded-Prefix สำหรับ redirect URLs +- Authentication flow ต้องการ cookie path ที่ถูกต้อง + +### Supabase Studio (`/supabase`) + +**สิ่งสำคัญ:** +```nginx +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection "upgrade"; +``` + +**ทำไม:** +- Supabase Studio ใช้ WebSocket สำหรับ real-time features +- ต้อง support HTTP/1.1 upgrade + +### Superset (`/superset`) + +**สิ่งสำคัญ:** +```nginx +proxy_connect_timeout 300s; +proxy_send_timeout 300s; +proxy_read_timeout 300s; +``` + +**ทำไม:** +- Dashboard queries อาจใช้เวลานาน +- ต้องการ timeout ที่สูงกว่าปกติ + +## 🐛 Troubleshooting + +### 502 Bad Gateway + +**สาเหตุ:** +- Backend service ไม่ทำงาน +- Network configuration ผิด + +**วิธีแก้:** +```bash +# ตรวจสอบ service +docker ps | grep + +# ตรวจสอบ network +docker network inspect shared_data_network + +# ดู logs +docker logs +docker logs nginx-proxy-manager +``` + +### 404 Not Found + +**สาเหตุ:** +- Path rewriting ไม่ถูกต้อง +- Backend ไม่ support subpath + +**วิธีแก้:** +```nginx +# ตรวจสอบ rewrite rule +rewrite ^/apiservice(/.*)$ $1 break; + +# ดู nginx logs +docker exec nginx-proxy-manager tail -f /data/logs/proxy-host-*.log +``` + +### Redirect Loop + +**สาเหตุ:** +- Cookie path ไม่ถูกต้อง +- X-Forwarded-* headers ขาดหาย + +**วิธีแก้:** +```nginx +# เพิ่ม headers +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header X-Forwarded-Host $host; + +# ตั้งค่า cookie path +proxy_cookie_path / /apiservice/; +``` + +### WebSocket Connection Failed + +**สาเหตุ:** +- ไม่มี WebSocket headers +- HTTP version ไม่ถูกต้อง + +**วิธีแก้:** +```nginx +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection "upgrade"; +``` + +### Session Lost After Refresh + +**สาเหตุ:** +- Cookie path ไม่ match กับ URL path +- SameSite cookie policy + +**วิธีแก้:** +```nginx +proxy_cookie_path / /apiservice/; +proxy_cookie_flags ~ secure samesite=lax; +``` + +## 🔐 Security Best Practices + +### 1. เปลี่ยน Default Password +``` +Settings → Users → Edit admin user +``` + +### 2. ใช้ HTTPS ใน Production +``` +SSL Tab → Request a new SSL Certificate +☑ Force SSL +☑ HSTS Enabled +``` + +### 3. ตั้งค่า Access Lists +``` +Access Lists → Add Access List +- Whitelist IP addresses +- Basic authentication +- Apply to sensitive services (Keycloak Admin, Superset) +``` + +### 4. Enable Rate Limiting +```nginx +# ใน Custom Nginx Configuration +limit_req_zone $binary_remote_addr zone=api_limit:10m rate=10r/s; +limit_req zone=api_limit burst=20 nodelay; +``` + +### 5. Hide Nginx Version +```nginx +# ใน Custom Nginx Configuration +server_tokens off; +``` + +## 📊 Monitoring + +### ดู Logs + +**ใน Nginx Proxy Manager UI:** +``` +Proxy Hosts → Click on host → Logs tab +``` + +**ใน Docker:** +```bash +# Nginx Proxy Manager logs +docker logs nginx-proxy-manager -f + +# Access logs +docker exec nginx-proxy-manager tail -f /data/logs/proxy-host-*.log + +# Error logs +docker exec nginx-proxy-manager tail -f /data/logs/error.log +``` + +### Health Checks + +```bash +# ตรวจสอบ Nginx Proxy Manager +curl http://192.168.100.9:8021 + +# ตรวจสอบ services ผ่าน proxy +curl http://ai.sriphat.com/apiservice/docs +curl http://ai.sriphat.com/supabase +curl http://ai.sriphat.com/keycloak +``` + +## 🔄 Backup & Restore + +### Backup Configuration + +```bash +# Backup Nginx Proxy Manager data +cd /path/to/01-infra +tar -czf npm-backup-$(date +%Y%m%d).tar.gz data/ + +# Backup specific configs +docker exec nginx-proxy-manager tar -czf /tmp/configs.tar.gz /data/nginx +docker cp nginx-proxy-manager:/tmp/configs.tar.gz ./npm-configs-backup.tar.gz +``` + +### Restore Configuration + +```bash +# Stop Nginx Proxy Manager +docker compose down + +# Restore data +tar -xzf npm-backup-YYYYMMDD.tar.gz + +# Start Nginx Proxy Manager +docker compose up -d +``` + +## 📚 Additional Resources + +- [Nginx Proxy Manager Docs](https://nginxproxymanager.com/guide/) +- [Nginx Reverse Proxy Guide](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) +- [Let's Encrypt SSL](https://letsencrypt.org/getting-started/) + +## 💡 Tips & Tricks + +### 1. Test Config Before Applying +```bash +# Test nginx config +docker exec nginx-proxy-manager nginx -t +``` + +### 2. Reload Without Restart +```bash +# Reload nginx (no downtime) +docker exec nginx-proxy-manager nginx -s reload +``` + +### 3. View Current Config +```bash +# View active nginx config +docker exec nginx-proxy-manager cat /etc/nginx/nginx.conf +``` + +### 4. Debug Mode +```nginx +# เพิ่มใน Custom Nginx Configuration +error_log /data/logs/error.log debug; +``` + +### 5. Custom Error Pages +```nginx +# เพิ่มใน Custom Nginx Configuration +error_page 502 503 504 /50x.html; +location = /50x.html { + root /usr/share/nginx/html; +} +``` + +## 🎯 Production Checklist + +- [ ] เปลี่ยน default admin password +- [ ] ตั้งค่า SSL certificate (Let's Encrypt) +- [ ] Enable Force SSL +- [ ] Enable HSTS +- [ ] ตั้งค่า Access Lists สำหรับ admin panels +- [ ] Enable rate limiting +- [ ] Hide server tokens +- [ ] ตั้งค่า backup schedule +- [ ] Test all services ผ่าน proxy +- [ ] Monitor logs สำหรับ errors +- [ ] Document custom configurations diff --git a/01-infra/nginx-configs/supabase-kong.conf b/01-infra/nginx-configs/supabase-kong.conf new file mode 100644 index 0000000..fc6e279 --- /dev/null +++ b/01-infra/nginx-configs/supabase-kong.conf @@ -0,0 +1,88 @@ +# Supabase Kong API Gateway - REST API +# Subpath: /supabase-api +# Backend: sdp-kong:8000 + +location /supabase-api { + # Remove /supabase-api prefix before forwarding + rewrite ^/supabase-api(/.*)$ $1 break; + + # Forward to Kong Gateway + proxy_pass http://sdp-kong:8000; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # API key and authorization headers + proxy_set_header Authorization $http_authorization; + proxy_set_header apikey $http_apikey; + + # CORS headers (if needed) + add_header Access-Control-Allow-Origin * always; + add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, PATCH, OPTIONS" always; + add_header Access-Control-Allow-Headers "Authorization, Content-Type, apikey, X-Client-Info" always; + + # Handle preflight requests + if ($request_method = 'OPTIONS') { + return 204; + } + + # Timeouts for API calls + proxy_connect_timeout 30s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + + # Disable buffering for streaming responses + proxy_buffering off; +} + +# REST API endpoints +location /supabase-api/rest { + rewrite ^/supabase-api(/.*)$ $1 break; + proxy_pass http://sdp-kong:8000; + proxy_set_header Host $host; + proxy_set_header Authorization $http_authorization; + proxy_set_header apikey $http_apikey; +} + +# Auth endpoints +location /supabase-api/auth { + rewrite ^/supabase-api(/.*)$ $1 break; + proxy_pass http://sdp-kong:8000; + proxy_set_header Host $host; + proxy_set_header Authorization $http_authorization; + proxy_set_header apikey $http_apikey; +} + +# Storage endpoints +location /supabase-api/storage { + rewrite ^/supabase-api(/.*)$ $1 break; + proxy_pass http://sdp-kong:8000; + proxy_set_header Host $host; + proxy_set_header Authorization $http_authorization; + proxy_set_header apikey $http_apikey; + + # Larger timeouts for file uploads + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + client_max_body_size 100M; +} + +# Realtime endpoints (WebSocket) +location /supabase-api/realtime { + rewrite ^/supabase-api(/.*)$ $1 break; + proxy_pass http://sdp-kong:8000; + proxy_set_header Host $host; + proxy_set_header Authorization $http_authorization; + proxy_set_header apikey $http_apikey; + + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400; +} diff --git a/01-infra/nginx-configs/supabase-studio.conf b/01-infra/nginx-configs/supabase-studio.conf new file mode 100644 index 0000000..9bb5c42 --- /dev/null +++ b/01-infra/nginx-configs/supabase-studio.conf @@ -0,0 +1,50 @@ +# Supabase Studio - Database Management UI +# Subpath: /supabase +# Backend: sdp-studio:3000 + +location /supabase { + # Remove /supabase prefix before forwarding + rewrite ^/supabase(/.*)$ $1 break; + + # Forward to Supabase Studio + proxy_pass http://sdp-studio:3000; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # WebSocket support for real-time features + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Timeouts + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # Disable buffering for real-time updates + proxy_buffering off; +} + +# API endpoints for Studio +location /supabase/api { + rewrite ^/supabase(/.*)$ $1 break; + proxy_pass http://sdp-studio:3000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} + +# Static assets +location /supabase/_next { + rewrite ^/supabase(/.*)$ $1 break; + proxy_pass http://sdp-studio:3000; + proxy_set_header Host $host; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; +} diff --git a/01-infra/nginx-configs/superset.conf b/01-infra/nginx-configs/superset.conf new file mode 100644 index 0000000..e91eb85 --- /dev/null +++ b/01-infra/nginx-configs/superset.conf @@ -0,0 +1,68 @@ +# Apache Superset - Business Intelligence +# Subpath: /superset +# Backend: superset:8088 + +location /superset { + # Remove /superset prefix before forwarding + rewrite ^/superset(/.*)$ $1 break; + + # Forward to Superset + proxy_pass http://superset:8088; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # Important for Superset subpath + proxy_set_header X-Script-Name /superset; + + # Session cookie handling + proxy_cookie_path / /superset/; + + # WebSocket support for real-time dashboards + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Timeouts (dashboards can take time to load) + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + + # Buffer settings for large responses + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; +} + +# API endpoints +location /superset/api { + rewrite ^/superset(/.*)$ $1 break; + proxy_pass http://superset:8088; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Script-Name /superset; + proxy_cookie_path / /superset/; +} + +# Static files +location /superset/static { + rewrite ^/superset(/.*)$ $1 break; + proxy_pass http://superset:8088; + proxy_set_header Host $host; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; +} + +# Superset assets +location /superset/superset { + rewrite ^/superset(/.*)$ $1 break; + proxy_pass http://superset:8088; + proxy_set_header Host $host; + proxy_set_header X-Script-Name /superset; +} diff --git a/03-apiservice/.env.example b/03-apiservice/.env.example index efc2d67..be9c4c8 100644 --- a/03-apiservice/.env.example +++ b/03-apiservice/.env.example @@ -32,3 +32,18 @@ ADMIN_PASSWORD=your-admin-password # API Key Encryption (for storing encrypted keys in DB) API_KEY_ENC_SECRET=your-encryption-secret-key-here + +# Debug settings (set to true for detailed logging, false for production) +DEBUG_AUTH=false + +# Keycloak Authentication (for web pages) +KEYCLOAK_SERVER_URL=http://keycloak:8080 +KEYCLOAK_REALM=master +KEYCLOAK_CLIENT_ID=apiservice +KEYCLOAK_CLIENT_SECRET=your-keycloak-client-secret +KEYCLOAK_REDIRECT_URI=http://localhost:8040/apiservice/auth/callback + +# Airflow Integration +AIRFLOW_API_URL=http://airflow-webserver:8080 +AIRFLOW_API_TOKEN=your-airflow-api-token +AIRFLOW_DAG_ID_FINANCE=process_finance_excel diff --git a/03-apiservice/.gitignore b/03-apiservice/.gitignore index 8d1428b..df43ee8 100644 --- a/03-apiservice/.gitignore +++ b/03-apiservice/.gitignore @@ -8,3 +8,5 @@ venv/ .mypy_cache/ ruff_cache/ .windsurf/ +data/uploads/* +!data/uploads/.gitkeep diff --git a/03-apiservice/AIRFLOW_INTEGRATION.md b/03-apiservice/AIRFLOW_INTEGRATION.md new file mode 100644 index 0000000..e1f1542 --- /dev/null +++ b/03-apiservice/AIRFLOW_INTEGRATION.md @@ -0,0 +1,334 @@ +# Airflow Integration Guide + +คู่มือการตั้งค่าและใช้งาน Airflow Integration สำหรับ Finance Excel Upload + +## 📋 Overview + +เมื่อ upload ไฟล์ Excel ผ่านหน้า `/data-management/finance` ระบบจะ: +1. บันทึกไฟล์ลง `/data/uploads/` +2. สร้าง record ใน database (`fastapi.upload_history`) +3. **Trigger Airflow DAG** `process_finance_excel` (พร้อม retry 3 ครั้ง, รอ 10 วิ) +4. Airflow จะ process ไฟล์และอัพเดท status กลับมา + +--- + +## 🔧 Configuration + +### 1. สร้าง Airflow API Token + +```bash +# เข้า Airflow container +docker exec -it airflow-webserver bash + +# สร้าง API token +airflow users create \ + --username apiservice \ + --firstname API \ + --lastname Service \ + --role Admin \ + --email apiservice@sriphat.com \ + --password your-secure-password + +# หรือใช้ CLI สร้าง token +python -c "from airflow.api.auth.backend.basic_auth import BasicAuth; print(BasicAuth().get_token('apiservice', 'your-secure-password'))" +``` + +**หรือใช้ Airflow UI:** +1. เข้า `http://ai.sriphat.com/airflow` +2. ไปที่ **Security** → **List Users** +3. เลือก user → **Edit** +4. Copy **API Token** (หรือสร้างใหม่) + +### 2. อัพเดท Environment Variables + +แก้ไข `.env` ใน `03-apiservice/`: + +```bash +# Airflow Integration +AIRFLOW_API_URL=http://airflow-webserver:8080 +AIRFLOW_API_TOKEN=your-airflow-api-token-here +AIRFLOW_DAG_ID_FINANCE=process_finance_excel +``` + +### 3. Restart API Service + +```bash +cd 03-apiservice +docker compose restart +``` + +--- + +## 📊 Database Schema Updates + +### New Fields in `fastapi.upload_history`: + +| Field | Type | Description | +|-------|------|-------------| +| `airflow_dag_run_id` | String | DAG run ID จาก Airflow | +| `airflow_state` | String | State: queued, running, success, failed | +| `processing_started_at` | DateTime | เวลาเริ่ม process | +| `processing_completed_at` | DateTime | เวลาเสร็จ | +| `error_message` | Text | Error message (ถ้ามี) | + +### Migration + +```bash +# Restart apiservice เพื่อสร้าง columns ใหม่ +cd 03-apiservice +docker compose down +docker compose up -d +``` + +--- + +## 🚀 How It Works + +### Upload Flow: + +``` +1. User uploads Excel file + ↓ +2. API saves file to /data/uploads/ + ↓ +3. Create UploadHistory record (status: pending) + ↓ +4. Trigger Airflow DAG (with retry logic) + ├─ Attempt 1: Try trigger + ├─ Wait 10 seconds + ├─ Attempt 2: Retry + ├─ Wait 10 seconds + └─ Attempt 3: Final retry + ↓ +5. Update record: + - Success: status=processing, airflow_dag_run_id=xxx + - Failure: status=error, error_message=xxx + ↓ +6. Return response to frontend +``` + +### Airflow DAG Flow: + +``` +validate_file + ↓ +process_data + ↓ +load_to_database + ↓ +update_status +``` + +--- + +## 📝 API Endpoints + +### Upload File (Triggers Airflow) + +```http +POST /data-management/finance/upload +Content-Type: multipart/form-data + +file: [Excel file] +description: "Optional description" +``` + +**Response:** +```json +{ + "success": true, + "message": "File 'finance.xlsx' uploaded successfully", + "upload_id": "upload_20260313_173000", + "filename": "20260313_173000_finance.xlsx", + "airflow_triggered": true, + "dag_run_id": "manual__2026-03-13T10:30:00+00:00", + "error": null +} +``` + +### Get Upload History + +```http +GET /data-management/finance/uploads +``` + +**Response:** +```json +[ + { + "id": "upload_20260313_173000", + "filename": "finance.xlsx", + "filepath": "/data/uploads/20260313_173000_finance.xlsx", + "uploaded_at": "2026-03-13T17:30:00", + "description": "Q1 2026 Finance Data", + "status": "processing", + "uploaded_by": "admin", + "airflow_dag_run_id": "manual__2026-03-13T10:30:00+00:00", + "airflow_state": "running", + "processing_started_at": "2026-03-13T17:30:05", + "processing_completed_at": null, + "error_message": null + } +] +``` + +--- + +## 🔍 Monitoring + +### 1. Check Upload Status + +```bash +# ดู logs ของ API Service +docker logs apiservice -f | grep "Airflow" + +# ตัวอย่าง logs: +# Triggering Airflow DAG (attempt 1/3) +# Airflow DAG triggered successfully: manual__2026-03-13T10:30:00+00:00 +``` + +### 2. Check Airflow DAG + +1. เข้า Airflow UI: `http://ai.sriphat.com/airflow` +2. ไปที่ **DAGs** → `process_finance_excel` +3. ดู **DAG Runs** และ **Task Instances** + +### 3. Check Database + +```sql +-- ดู upload history พร้อม Airflow status +SELECT + upload_id, + filename, + status, + airflow_dag_run_id, + airflow_state, + processing_started_at, + processing_completed_at, + error_message +FROM fastapi.upload_history +ORDER BY uploaded_at DESC +LIMIT 10; +``` + +--- + +## 🐛 Troubleshooting + +### Issue: "Failed to trigger Airflow after 3 attempts" + +**สาเหตุ:** +- Airflow API Token ไม่ถูกต้อง +- Airflow webserver ไม่ทำงาน +- Network connection ล้มเหลว + +**วิธีแก้:** + +```bash +# 1. ตรวจสอบ Airflow webserver +docker ps | grep airflow-webserver + +# 2. ตรวจสอบ network +docker network inspect shared_data_network + +# 3. ทดสอบ API token +curl -X GET \ + http://airflow-webserver:8080/api/v1/dags \ + -H "Authorization: Bearer YOUR_TOKEN" + +# 4. ดู logs +docker logs airflow-webserver -f +docker logs apiservice -f +``` + +### Issue: DAG ไม่ปรากฏใน Airflow UI + +**สาเหตุ:** +- DAG file มี syntax error +- DAG ถูก pause +- Airflow scheduler ไม่ทำงาน + +**วิธีแก้:** + +```bash +# 1. ตรวจสอบ DAG file +docker exec airflow-webserver airflow dags list | grep process_finance_excel + +# 2. Test DAG +docker exec airflow-webserver airflow dags test process_finance_excel + +# 3. Unpause DAG +docker exec airflow-webserver airflow dags unpause process_finance_excel + +# 4. ตรวจสอบ scheduler +docker logs airflow-scheduler -f +``` + +### Issue: DAG triggered แต่ไม่ทำงาน + +**สาเหตุ:** +- Volume mount ไม่ถูกต้อง (Airflow เข้าถึง `/data/uploads` ไม่ได้) +- Dependencies ขาดหาย (pandas, openpyxl) + +**วิธีแก้:** + +```bash +# 1. ตรวจสอบ volume +docker exec airflow-worker ls -la /data/uploads + +# 2. ติดตั้ง dependencies +docker exec airflow-worker pip install pandas openpyxl + +# หรือเพิ่มใน docker-compose.yaml: +# _PIP_ADDITIONAL_REQUIREMENTS: "pandas openpyxl" +``` + +--- + +## 🔐 Security Notes + +1. **API Token Security:** + - เก็บ token ใน `.env` (ไม่ commit ลง git) + - ใช้ token ที่มี scope จำกัด (ไม่ใช่ admin token) + - Rotate token เป็นระยะ + +2. **File Access:** + - Airflow ต้องมี read access ไปยัง `/data/uploads` + - ตรวจสอบ file permissions + +3. **Network:** + - API Service และ Airflow ต้องอยู่ใน `shared_data_network` + - ใช้ internal network (ไม่ expose Airflow API ออกภายนอก) + +--- + +## 📚 References + +- [Airflow REST API Documentation](https://airflow.apache.org/docs/apache-airflow/stable/stable-rest-api-ref.html) +- [Airflow Authentication](https://airflow.apache.org/docs/apache-airflow/stable/security/api.html) +- [FastAPI Background Tasks](https://fastapi.tiangolo.com/tutorial/background-tasks/) + +--- + +## 🎯 Next Steps + +1. **Implement DAG Logic:** + - แก้ไข `05-airflow/dags/process_finance_excel.py` + - เพิ่ม data transformation logic + - เพิ่ม database loading logic + +2. **Add Callback:** + - สร้าง endpoint `/api/v1/airflow/callback/{upload_id}` + - Airflow จะ callback เมื่อเสร็จ/ล้มเหลว + - อัพเดท `processing_completed_at` และ `airflow_state` + +3. **Frontend Updates:** + - เพิ่ม real-time status polling + - แสดง Airflow state badges + - Link ไปยัง Airflow DAG run + +4. **Testing:** + - ทดสอบ upload file + - ตรวจสอบ Airflow trigger + - ตรวจสอบ retry logic + - ทดสอบ error handling diff --git a/03-apiservice/KEYCLOAK-SETUP.md b/03-apiservice/KEYCLOAK-SETUP.md new file mode 100644 index 0000000..09cb708 --- /dev/null +++ b/03-apiservice/KEYCLOAK-SETUP.md @@ -0,0 +1,325 @@ +# Keycloak Setup Guide for API Service Authentication + +## Prerequisites + +1. Keycloak must be running at `http://localhost:8085` +2. PostgreSQL database must have `keycloak` database created +3. API Service must have Keycloak configuration in `.env` + +## Step 1: Restart Infrastructure to Create Keycloak Database + +```bash +cd 01-infra +docker compose down +docker compose --env-file ../.env.global up -d +``` + +Wait for Keycloak to initialize (check logs): +```bash +docker logs keycloak -f +``` + +Look for: `Keycloak ... started` + +## Step 2: Access Keycloak Admin Console + +1. Open browser: `http://localhost:8085/admin` +2. Login credentials: + - **Username**: `admin` + - **Password**: `admin_secret_pass_2026` + +## Step 3: Create Client for API Service + +### 3.1 Navigate to Clients +1. In left sidebar, click **Clients** +2. Click **Create client** button + +### 3.2 General Settings +- **Client type**: `OpenID Connect` +- **Client ID**: `apiservice` +- Click **Next** + +### 3.3 Capability Config +- **Client authentication**: `ON` (toggle to enable) +- **Authorization**: `OFF` +- **Authentication flow**: + - ✅ Standard flow + - ✅ Direct access grants + - ❌ Implicit flow (uncheck) + - ❌ Service accounts roles (uncheck) +- Click **Next** + +### 3.4 Login Settings +Fill in the following URLs: + +**Root URL**: +``` +http://localhost:8040 +``` + +**Home URL**: +``` +http://localhost:8040/apiservice/ +``` + +**Valid redirect URIs** (add both): +``` +http://localhost:8040/apiservice/auth/callback +https://ai.sriphat.com/apiservice/auth/callback +``` + +**Valid post logout redirect URIs** (add both): +``` +http://localhost:8040/apiservice/ +https://ai.sriphat.com/apiservice/ +``` + +**Web origins** (add both): +``` +http://localhost:8040 +https://ai.sriphat.com +``` + +Click **Save** + +### 3.5 Get Client Secret +1. Go to **Credentials** tab +2. Copy the **Client secret** value +3. Save it - you'll need it for the `.env` file + +## Step 4: Configure API Service + +### 4.1 Create/Update `.env` file in `03-apiservice` + +```bash +cd ../03-apiservice +``` + +Create `.env` file with: + +```bash +# Application +APP_NAME=APIsService +ROOT_PATH=/apiservice + +# Timezone +TIMEZONE=Asia/Bangkok + +# PostgreSQL Database +DB_HOST=postgres +DB_PORT=5432 +DB_USER=postgres +DB_PASSWORD=Secure_Hospital_Pass_2026 +DB_NAME=postgres +DB_SSLMODE=prefer + +# Supabase (if not using, keep dummy values) +SUPABASE_DB_HOST=localhost +SUPABASE_DB_PORT=5432 +SUPABASE_DB_USER=postgres +SUPABASE_DB_PASSWORD=dummy +SUPABASE_DB_NAME=postgres +SUPABASE_DB_SSLMODE=disable +SUPABASE_API_URL=http://localhost:8000 +SUPABASE_API_KEY=dummy + +# Admin Authentication +ADMIN_SECRET_KEY=apiservice_admin_secret_2026 +ADMIN_USERNAME=admin +ADMIN_PASSWORD=change_me_2026 + +# API Key Encryption +API_KEY_ENC_SECRET=dev_encryption_secret_2026 + +# Keycloak Authentication +KEYCLOAK_SERVER_URL=http://keycloak:8080 +KEYCLOAK_REALM=master +KEYCLOAK_CLIENT_ID=apiservice +KEYCLOAK_CLIENT_SECRET= +KEYCLOAK_REDIRECT_URI=http://localhost:8040/apiservice/auth/callback +``` + +**Important**: Replace `` with the actual secret from Step 3.5 + +## Step 5: Create Test User in Keycloak + +### 5.1 Create User +1. In Keycloak Admin, go to **Users** +2. Click **Add user** +3. Fill in: + - **Username**: `testuser` + - **Email**: `test@sriphat.local` + - **First name**: `Test` + - **Last name**: `User` + - **Email verified**: `ON` (toggle) +4. Click **Create** + +### 5.2 Set Password +1. Go to **Credentials** tab +2. Click **Set password** +3. Fill in: + - **Password**: `password` + - **Password confirmation**: `password` + - **Temporary**: `OFF` (toggle off) +4. Click **Save** +5. Confirm by clicking **Save password** + +## Step 6: Rebuild and Restart API Service + +```bash +cd 03-apiservice +docker compose down +docker compose --env-file ../.env up --build -d +``` + +Check logs: +```bash +docker logs apiservice -f +``` + +Look for successful startup without errors. + +## Step 7: Test Authentication Flow + +### 7.1 Access Landing Page +1. Open browser: `http://localhost:8040/apiservice/` +2. You should be redirected to Keycloak login page + +### 7.2 Login +1. Enter credentials: + - **Username**: `testuser` + - **Password**: `password` +2. Click **Sign In** + +### 7.3 Verify Success +1. You should be redirected back to the landing page +2. You should see your name in the top-right corner: "👤 Test User" +3. You should see a **Logout** button + +### 7.4 Test Protected Pages +Try accessing: +- `/apiservice/` - Landing page (requires auth) +- `/apiservice/docs` - API docs (requires auth) +- `/apiservice/data-management/finance` - Finance page (requires auth) + +All should work without redirecting to login again. + +### 7.5 Test API Endpoints (Should NOT require Keycloak auth) +```bash +# API endpoints use API Key authentication, not Keycloak +curl -X GET http://localhost:8040/apiservice/api/v1/health +``` + +This should work without Keycloak authentication. + +### 7.6 Test Logout +1. Click **Logout** button +2. You should be logged out from Keycloak +3. Accessing any protected page should redirect to login again + +## Troubleshooting + +### Error: "access_denied" in Keycloak logs +**Cause**: Client not configured correctly or client secret mismatch + +**Solution**: +1. Verify client ID is exactly `apiservice` +2. Verify client secret in `.env` matches Keycloak +3. Verify redirect URIs are correct +4. Restart API service after changing `.env` + +### Error: "SessionMiddleware must be installed" +**Cause**: Middleware order is incorrect + +**Solution**: Already fixed in `app/main.py` - SessionMiddleware is added after WebAuthenticationMiddleware + +### Error: "Invalid redirect_uri" +**Cause**: Redirect URI not in Keycloak's allowed list + +**Solution**: +1. Go to Keycloak → Clients → apiservice → Settings +2. Add the exact redirect URI to "Valid redirect URIs" +3. Make sure it matches `KEYCLOAK_REDIRECT_URI` in `.env` + +### Error: Connection refused to Keycloak +**Cause**: Keycloak not running or wrong URL + +**Solution**: +1. Check Keycloak is running: `docker ps | grep keycloak` +2. Verify `KEYCLOAK_SERVER_URL=http://keycloak:8080` (use container name, not localhost) +3. Verify both services are on same Docker network + +## Production Deployment + +For production deployment: + +1. **Use HTTPS**: + ```bash + KEYCLOAK_REDIRECT_URI=https://ai.sriphat.com/apiservice/auth/callback + ``` + +2. **Update Keycloak Client**: + - Add production redirect URIs + - Remove localhost URIs + +3. **Secure Cookies**: + - SessionMiddleware should use `secure=True, httponly=True, samesite='lax'` + +4. **Use Nginx Proxy Manager**: + - Configure SSL certificates + - Set up proper reverse proxy rules + +5. **Environment Variables**: + - Use Docker secrets or external secret management + - Never commit `.env` with real secrets to git + +## Security Notes + +1. **Client Secret**: Keep this secret! Never commit to git +2. **Session Secret**: Use strong random value for `ADMIN_SECRET_KEY` +3. **HTTPS**: Always use HTTPS in production +4. **Token Expiration**: Configure appropriate token lifetimes in Keycloak +5. **CORS**: Restrict `allow_origins` to known domains only + +## Architecture + +``` +User Browser + ↓ + ↓ (1) Access protected page + ↓ +API Service (FastAPI) + ↓ + ↓ (2) Check session - not authenticated + ↓ + ↓ (3) Redirect to Keycloak login + ↓ +Keycloak (OAuth2/OIDC) + ↓ + ↓ (4) User enters credentials + ↓ + ↓ (5) Redirect back with auth code + ↓ +API Service + ↓ + ↓ (6) Exchange code for tokens + ↓ + ↓ (7) Store user info in session + ↓ + ↓ (8) Redirect to original page + ↓ +User sees protected page with user info +``` + +## API vs Web Authentication + +| Type | Authentication Method | Protected Routes | +|------|----------------------|------------------| +| **Web Pages** | Keycloak (OAuth2/OIDC) | `/`, `/docs`, `/data-management/*` | +| **API Endpoints** | API Key (Bearer Token) | `/api/v1/*` | +| **Admin Panel** | SQLAdmin (Basic Auth) | `/admin/*` | + +This separation allows: +- Human users to login via Keycloak for web UI +- Applications/services to use API Keys for programmatic access +- Admins to manage API keys via separate admin panel diff --git a/03-apiservice/README-PAGES.md b/03-apiservice/README-PAGES.md new file mode 100644 index 0000000..c13e225 --- /dev/null +++ b/03-apiservice/README-PAGES.md @@ -0,0 +1,226 @@ +# Web Pages Documentation + +## Overview + +The API service now includes web pages for easy access to all platform services and data management functionality. + +## Pages + +### 1. Landing Page (`/`) + +Main dashboard with navigation to all services: +- **Supabase**: PostgreSQL database with REST API +- **API Docs**: Interactive API documentation +- **Airflow**: Workflow orchestration +- **Airbyte**: Data integration and ETL +- **Data Management**: File upload and processing (with submenu) +- **DBT**: Data transformation +- **Superset**: Business intelligence and visualization + +### 2. Finance Excel Upload (`/data-management/finance`) + +Upload and process financial Excel files with the following features: + +**Upload Form:** +- Drag-and-drop or click to select Excel files (.xlsx, .xls) +- Optional description field +- File validation and size display + +**Upload Processing:** +- Files saved to `/data/uploads/` with timestamp prefix +- Unique filename generation to prevent conflicts +- Automatic Airflow job triggering (when configured) + +**Upload History:** +- Real-time list of all uploaded files +- Status tracking: pending, processing, success, error +- Expandable accordion for logs and error details +- Auto-refresh every 10 seconds + +## API Endpoints + +### Page Routes + +``` +GET / - Landing page +GET /data-management/finance - Finance upload page +POST /data-management/finance/upload - Upload file endpoint +GET /data-management/finance/uploads - Get all uploads +GET /data-management/finance/uploads/{upload_id} - Get specific upload +``` + +### Upload Endpoint + +**POST** `/data-management/finance/upload` + +**Form Data:** +- `file`: Excel file (.xlsx or .xls) +- `description`: Optional description text + +**Response:** +```json +{ + "success": true, + "message": "File 'example.xlsx' uploaded successfully", + "upload_id": "upload_20260306_174500", + "filename": "20260306_174500_example.xlsx" +} +``` + +**Error Response:** +```json +{ + "detail": "Invalid file type. Only .xlsx and .xls files are allowed." +} +``` + +## File Storage + +Uploaded files are stored in: +``` +/data/uploads/YYYYMMDD_HHMMSS_original_filename.xlsx +``` + +Example: +``` +/data/uploads/20260306_174530_finance_report.xlsx +``` + +## Airflow Integration + +The upload endpoint includes a placeholder for Airflow integration. To enable: + +1. **Configure Airflow endpoint** in `app/routes/pages.py`: + ```python + airflow_url = "http://airflow-webserver:8080/api/v1/dags/{dag_id}/dagRuns" + ``` + +2. **Set DAG ID** for the finance processing job + +3. **Uncomment the Airflow trigger code** in `trigger_airflow_job()` function + +4. **Configure authentication** (username/password or API token) + +### Example Airflow Integration + +```python +async def trigger_airflow_job(filepath: str, upload_id: str) -> str: + import httpx + + airflow_url = "http://airflow-webserver:8080/api/v1/dags/finance_processing/dagRuns" + headers = {"Content-Type": "application/json"} + auth = ("airflow", "airflow") + + payload = { + "conf": { + "filepath": filepath, + "upload_id": upload_id + } + } + + async with httpx.AsyncClient() as client: + response = await client.post( + airflow_url, + json=payload, + headers=headers, + auth=auth + ) + response.raise_for_status() + result = response.json() + return result["dag_run_id"] +``` + +## Upload Status Tracking + +Upload records include: +- `id`: Unique identifier +- `filename`: Original filename +- `filepath`: Full path to saved file +- `uploaded_at`: ISO timestamp +- `description`: User-provided description +- `status`: pending | processing | success | error +- `job_id`: Airflow DAG run ID (when triggered) +- `logs`: Error logs or processing information + +## Database Storage + +Currently uses in-memory storage (`uploads_db` list). For production: + +1. **Create database table:** + ```sql + CREATE TABLE uploads ( + id VARCHAR(100) PRIMARY KEY, + filename VARCHAR(255) NOT NULL, + filepath VARCHAR(500) NOT NULL, + uploaded_at TIMESTAMP NOT NULL, + description TEXT, + status VARCHAR(20) NOT NULL, + job_id VARCHAR(100), + logs TEXT, + created_at TIMESTAMP DEFAULT NOW() + ); + ``` + +2. **Replace in-memory storage** with SQLAlchemy model + +3. **Add database queries** in route handlers + +## Access URLs + +- **Local**: http://localhost:8040/apiservice/ +- **Production**: https://ai.sriphat.com/apiservice/ + +## Security Considerations + +1. **File validation**: Only .xlsx and .xls files allowed +2. **Filename sanitization**: Spaces replaced with underscores +3. **Unique filenames**: Timestamp prefix prevents conflicts +4. **File size limits**: Configure in nginx/FastAPI if needed +5. **Authentication**: Add authentication middleware for production + +## Future Enhancements + +1. **Database persistence**: Replace in-memory storage +2. **File preview**: Show Excel data preview before processing +3. **Batch upload**: Support multiple file uploads +4. **Download processed files**: Allow downloading results +5. **User management**: Track uploads by user +6. **Email notifications**: Notify on processing completion +7. **Scheduled jobs**: Automatic periodic processing +8. **Data validation**: Validate Excel structure before processing + +## Troubleshooting + +### Upload fails with 500 error +- Check `/data/uploads/` directory exists and is writable +- Verify volume mount in docker-compose.yml +- Check container logs: `docker logs apiservice` + +### Files not persisting after container restart +- Ensure volume mount is configured: `./data/uploads:/data/uploads` +- Check file permissions on host directory + +### Airflow job not triggering +- Verify Airflow is accessible from container +- Check network connectivity: `docker network inspect shared_data_network` +- Verify Airflow credentials and DAG ID +- Check logs in upload history + +## Development + +To add new data management pages: + +1. Create HTML template in `app/templates/` +2. Add route in `app/routes/pages.py` +3. Update landing page menu in `index.html` +4. Add submenu item if needed + +Example: +```python +@router.get("/data-management/hr", response_class=HTMLResponse) +async def hr_page(request: Request): + return templates.TemplateResponse( + "data_management_hr.html", + {"request": request, "root_path": settings.ROOT_PATH} + ) +``` diff --git a/03-apiservice/app/api/v1/routes.py b/03-apiservice/app/api/v1/routes.py index bbda5ad..469ee79 100644 --- a/03-apiservice/app/api/v1/routes.py +++ b/03-apiservice/app/api/v1/routes.py @@ -9,9 +9,9 @@ from fastapi import APIRouter, Depends from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session -from app.api.v1.schemas import FeedCheckpointIn, FeedWaitingTimeIn +from app.api.v1.schemas import FeedCheckpointIn, FeedWaitingTimeIn, PatientAppointmentIn from app.core.config import settings -from app.db.models import RawOpdCheckpoint, RawWaitingTime +from app.db.models import RawOpdCheckpoint, RawWaitingTime, PatientAppointment from app.security.dependencies import get_db, require_permission from app.utils.supabase_client import SupabaseAPIError, upsert_to_supabase_sync @@ -21,6 +21,7 @@ router = APIRouter(prefix="/api/v1") PERM_FEED_CHECKPOINT_WRITE = "feed.checkpoint:write" PERM_FEED_OLD_CHECKPOINT_WRITE = "feed.old-checkpoint:write" +PERM_FEED_PATIENT_APPOINTMENT_WRITE = "feed.patient-appointment:write" def _to_tz(dt): @@ -220,3 +221,86 @@ def upsert_opd_checkpoint( "error": supabase_error, }, } + + +@router.post("/feed/patient-appointment") +def upsert_patient_appointment( + payload: list[PatientAppointmentIn], + _: Annotated[object, Depends(require_permission(PERM_FEED_PATIENT_APPOINTMENT_WRITE))], + db: Annotated[Session, Depends(get_db)], +): + rows = [] + supabase_rows = [] + + for item in payload: + # Prepare data for local database + row = { + "hn": item.hn, + "txn": item.txn, + "date": item.date, #+' 0:00:00' + "time": _to_tz(datetime.strptime(item.date.strftime("%Y-%m-%d")+' '+item.time,'%Y-%m-%d %H:%M:%S')), + "doctor_code": item.doctor_code, + "period": item.period, + "appointment_type": item.appointment_type, + "updated_at": datetime.now(ZoneInfo(settings.TIMEZONE)), + } + rows.append(row) + + # Prepare data for Supabase API (convert datetime to ISO string) + supabase_row = { + "hn": item.hn, + "txn": item.txn, + "date": _to_iso(_to_tz(datetime.strptime(item.date.strftime("%Y-%m-%d")+' 00:00:00+07:00','%Y-%m-%d %H:%M:%S%z'))), + "time": _to_iso(_to_tz(datetime.strptime(item.date.strftime("%Y-%m-%d")+' '+item.time+'+07:00','%Y-%m-%d %H:%M:%S%z'))), + "doctor_code": item.doctor_code, + "period": item.period, + "appointment_type": item.appointment_type, + "updated_at": datetime.now(ZoneInfo(settings.TIMEZONE)).isoformat(), + } + supabase_rows.append(supabase_row) + + # Insert/update to local database + stmt = insert(PatientAppointment).values(rows) + update_cols = { + "txn": stmt.excluded.txn, + "doctor_code": stmt.excluded.doctor_code, + "period": stmt.excluded.period, + "appointment_type": stmt.excluded.appointment_type, + "updated_at": stmt.excluded.updated_at, + } + + stmt = stmt.on_conflict_do_update( + index_elements=[PatientAppointment.hn, PatientAppointment.date, PatientAppointment.time], + set_=update_cols, + ) + result = db.execute(stmt) + db.commit() + + # Send data to Supabase via API call + supabase_result = None + supabase_error = None + + try: + logger.info(f"Sending {len(supabase_rows)} patient appointment records to Supabase API") + supabase_result = upsert_to_supabase_sync( + table="patient_appointment", + data=supabase_rows, + on_conflict="hn,date,time", + ) + logger.info(f"Successfully sent patient appointment data to Supabase: {supabase_result.get('status_code')}") + except SupabaseAPIError as e: + logger.error(f"Failed to send patient appointment data to Supabase: {str(e)}") + supabase_error = str(e) + except Exception as e: + logger.error(f"Unexpected error sending patient appointment data to Supabase: {str(e)}") + supabase_error = f"Unexpected error: {str(e)}" + + return { + "upserted": len(rows), + "rowcount": result.rowcount, + "supabase": { + "success": supabase_result is not None, + "result": supabase_result, + "error": supabase_error, + }, + } diff --git a/03-apiservice/app/api/v1/schemas.py b/03-apiservice/app/api/v1/schemas.py index 2d1b3ed..76db03f 100644 --- a/03-apiservice/app/api/v1/schemas.py +++ b/03-apiservice/app/api/v1/schemas.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, time, date from pydantic import BaseModel @@ -27,3 +27,13 @@ class FeedCheckpointIn(BaseModel): timestamp_out: datetime | None = None waiting_time: int | None = None bu: str | None = None + + +class PatientAppointmentIn(BaseModel): + hn: str + txn: int | None = None + date: date + time: str + doctor_code: str | None = None + period: str | None = None + appointment_type: str | None = None diff --git a/03-apiservice/app/core/config.py b/03-apiservice/app/core/config.py index 7de53af..6c5555d 100644 --- a/03-apiservice/app/core/config.py +++ b/03-apiservice/app/core/config.py @@ -33,5 +33,20 @@ class Settings(BaseSettings): API_KEY_ENC_SECRET: str | None = None + # Debug settings + DEBUG_AUTH: bool = False # Set to True to enable detailed authentication logging + + # Keycloak Authentication (for web pages only) + KEYCLOAK_SERVER_URL: str = "http://keycloak:8080" + KEYCLOAK_REALM: str = "master" + KEYCLOAK_CLIENT_ID: str = "apiservice" + KEYCLOAK_CLIENT_SECRET: str = "" + KEYCLOAK_REDIRECT_URI: str = "http://localhost:8040/apiservice/auth/callback" + + # Airflow Integration + AIRFLOW_API_URL: str = "http://airflow-webserver:8080" + AIRFLOW_API_TOKEN: str = "" + AIRFLOW_DAG_ID_FINANCE: str = "process_finance_excel" + settings = Settings() diff --git a/03-apiservice/app/db/init_db.py b/03-apiservice/app/db/init_db.py index 3b2887d..1c20527 100644 --- a/03-apiservice/app/db/init_db.py +++ b/03-apiservice/app/db/init_db.py @@ -1,12 +1,54 @@ from sqlalchemy import text +from sqlalchemy.orm import Session +import logging from app.db.base import Base from app.db.engine import engine +from app.models.user import User, Role # Import models to ensure they're registered + +logger = logging.getLogger(__name__) def init_db() -> None: + """Initialize database schemas and tables""" with engine.begin() as conn: + # Create schemas conn.execute(text("CREATE SCHEMA IF NOT EXISTS fastapi")) conn.execute(text("CREATE SCHEMA IF NOT EXISTS operationbi")) - + conn.execute(text("CREATE SCHEMA IF NOT EXISTS rawdata")) + + # Create all tables Base.metadata.create_all(bind=conn) + logger.info("Database schemas and tables created") + + # Seed default roles + seed_roles() + + +def seed_roles() -> None: + """Seed default roles if they don't exist""" + from app.db.session import SessionLocal + + db = SessionLocal() + try: + roles_data = [ + {"name": "admin", "description": "Full system access - can manage users and access all features"}, + {"name": "operation", "description": "Data management access - can upload and manage data"} + ] + + for role_data in roles_data: + existing = db.query(Role).filter(Role.name == role_data["name"]).first() + if not existing: + role = Role(**role_data) + db.add(role) + logger.info(f"Created role: {role_data['name']}") + else: + logger.info(f"Role already exists: {role_data['name']}") + + db.commit() + logger.info("Role seeding completed") + except Exception as e: + logger.error(f"Error seeding roles: {e}") + db.rollback() + finally: + db.close() diff --git a/03-apiservice/app/db/models.py b/03-apiservice/app/db/models.py index d596918..273068c 100644 --- a/03-apiservice/app/db/models.py +++ b/03-apiservice/app/db/models.py @@ -46,6 +46,25 @@ class RawWaitingTime(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) +class PatientAppointment(Base): + __tablename__ = "patient_appointment" + __table_args__ = ( + UniqueConstraint("hn", "date", "time", name="uq_patient_appointment_hn_date_time"), + {"schema": "rawdata"}, + ) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + hn: Mapped[str] = mapped_column(String(50), nullable=False) + txn: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + date: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + doctor_code: Mapped[str | None] = mapped_column(String(50), nullable=True) + period: Mapped[str | None] = mapped_column(String(50), nullable=True) + appointment_type: Mapped[str | None] = mapped_column(String(100), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + class ApiClient(Base): __tablename__ = "api_client" __table_args__ = {"schema": "fastapi"} diff --git a/03-apiservice/app/db/session.py b/03-apiservice/app/db/session.py new file mode 100644 index 0000000..6b82ee3 --- /dev/null +++ b/03-apiservice/app/db/session.py @@ -0,0 +1,27 @@ +""" +Database session management +""" +from sqlalchemy.orm import sessionmaker, Session +from app.db.engine import engine + +# Create SessionLocal class +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + +def get_db() -> Session: + """ + Dependency to get database session + + Usage: + @app.get("/items") + def read_items(db: Session = Depends(get_db)): + ... + + Yields: + Database session + """ + db = SessionLocal() + try: + yield db + finally: + db.close() diff --git a/03-apiservice/app/main.py b/03-apiservice/app/main.py index ebe9878..bd49e54 100644 --- a/03-apiservice/app/main.py +++ b/03-apiservice/app/main.py @@ -12,6 +12,10 @@ import sqladmin from app.admin import mount_admin from app.api.v1.routes import router as v1_router +from app.routes.pages import router as pages_router +from app.routes.auth import router as auth_router +from app.routes.admin_users import router as admin_users_router +from app.middleware.auth_middleware import WebAuthenticationMiddleware from app.core.config import settings from app.db.init_db import init_db @@ -80,8 +84,9 @@ async def global_exception_handler(request, exc): status_code=500, content={"detail": "Internal server error", "error": str(exc)} ) +# Middleware order is important! They execute in reverse order (LIFO) +# WebAuthenticationMiddleware needs SessionMiddleware, so SessionMiddleware must be added AFTER app.add_middleware(ForceHTTPSMiddleware) -app.add_middleware(SessionMiddleware, secret_key=settings.ADMIN_SECRET_KEY) app.add_middleware(ForwardedProtoMiddleware) app.add_middleware( CORSMiddleware, @@ -90,7 +95,17 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) -app.include_router(v1_router) +# Add web authentication middleware (protects /, /docs, /data-management/* only) +# API endpoints (/api/v1/*) continue to use API Key authentication +app.add_middleware(WebAuthenticationMiddleware) +# SessionMiddleware must be added AFTER middlewares that use it (due to LIFO execution) +app.add_middleware(SessionMiddleware, secret_key=settings.ADMIN_SECRET_KEY) + +app.include_router(v1_router) # API endpoints - use API Key auth +app.include_router(pages_router) # Web pages - use Keycloak auth +app.include_router(auth_router) # Authentication routes +app.include_router(admin_users_router) # Admin user management API + app.mount("/admin/statics", StaticFiles(directory=statics_path), name="admin_statics") app.mount("/apiservice/admin/statics", StaticFiles(directory=statics_path), name="proxy_admin_statics") mount_admin(app) diff --git a/03-apiservice/app/middleware/__init__.py b/03-apiservice/app/middleware/__init__.py new file mode 100644 index 0000000..93fa6f7 --- /dev/null +++ b/03-apiservice/app/middleware/__init__.py @@ -0,0 +1 @@ +# Middleware package diff --git a/03-apiservice/app/middleware/auth_middleware.py b/03-apiservice/app/middleware/auth_middleware.py new file mode 100644 index 0000000..bfbbbea --- /dev/null +++ b/03-apiservice/app/middleware/auth_middleware.py @@ -0,0 +1,169 @@ +""" +Authentication middleware for web pages +Protects web UI routes while allowing API endpoints to use API Key auth +""" +from fastapi import Request +from fastapi.responses import RedirectResponse +from starlette.middleware.base import BaseHTTPMiddleware +from app.core.config import settings +import logging + +logger = logging.getLogger(__name__) + + +class WebAuthenticationMiddleware(BaseHTTPMiddleware): + """ + Middleware to enforce Keycloak authentication for web pages + + Protected routes (require user login): + - / (landing page) + - /docs (API documentation) + - /data-management/* (data management pages) + + Excluded routes (no auth required or use different auth): + - /auth/* (authentication endpoints) + - /api/v1/* (API endpoints - use API Key auth) + - /admin/* (SQLAdmin - has its own auth) + - /apiservice/admin/statics/* (admin static files) + - /admin/statics/* (admin static files) + """ + + # Routes that require user authentication + PROTECTED_PATHS = [ + "/", + "/docs", + "/redoc", + "/openapi.json", + "/data-management" + ] + + # Routes that are excluded from user authentication + EXCLUDED_PATHS = [ + "/auth", # Authentication endpoints + "/api/v1", # API endpoints (use API Key) + "/admin", # SQLAdmin (has own auth) + ] + + async def dispatch(self, request: Request, call_next): + """Process request and check authentication if needed""" + + # Get the path without root_path prefix + path = request.url.path + + # Remove root_path prefix if it exists + if settings.ROOT_PATH and path.startswith(settings.ROOT_PATH): + path = path[len(settings.ROOT_PATH):] + + # Ensure path starts with / + if not path.startswith("/"): + path = "/" + path + + # Check if path is excluded from authentication + is_excluded = any(path.startswith(excluded) for excluded in self.EXCLUDED_PATHS) + + if is_excluded: + # Skip authentication for excluded paths (including /api/v1/*) + return await call_next(request) + + # Check if path requires authentication + requires_auth = any( + path == protected or path.startswith(protected + "/") + for protected in self.PROTECTED_PATHS + ) + + if requires_auth: + # Check if user is authenticated + user = request.session.get("user") + + if not user: + # User not authenticated - redirect to login + # Preserve the original path for redirect after login + original_path = request.url.path + + # Build login URL with redirect + login_url = f"{settings.ROOT_PATH}/auth/login?redirect_to={original_path}" + + logger.info(f"Unauthenticated access to {original_path}, redirecting to login") + + return RedirectResponse(url=login_url, status_code=302) + + # Role-based access control for specific paths + user_roles = user.get("roles", []) + + # /data-management/* requires 'operation' or 'admin' role + if path.startswith("/data-management"): + if not any(role in user_roles for role in ["admin", "operation"]): + logger.warning( + f"Access denied: User {user.get('username')} " + f"(roles: {user_roles}) tried to access {path}" + ) + # Return 403 Forbidden page + from fastapi.responses import HTMLResponse + return HTMLResponse( + content=f""" + + + + Access Denied + + + +
+

🚫

+

Access Denied

+

You don't have permission to access this page.

+
+

Required role: operation or admin

+

Your roles: {', '.join(user_roles) if user_roles else 'None'}

+
+

Please contact your administrator if you need access.

+ ← Go to Home +
+ + + """, + status_code=403 + ) + + # Continue with request + return await call_next(request) diff --git a/03-apiservice/app/models/__init__.py b/03-apiservice/app/models/__init__.py new file mode 100644 index 0000000..a025be8 --- /dev/null +++ b/03-apiservice/app/models/__init__.py @@ -0,0 +1,7 @@ +""" +Database models package +""" +from app.models.user import User, Role +from app.models.upload import UploadHistory + +__all__ = ["User", "Role", "UploadHistory"] diff --git a/03-apiservice/app/models/upload.py b/03-apiservice/app/models/upload.py new file mode 100644 index 0000000..1089a69 --- /dev/null +++ b/03-apiservice/app/models/upload.py @@ -0,0 +1,33 @@ +""" +Upload History model for tracking file uploads +""" +from sqlalchemy import Column, Integer, String, DateTime, Text +from sqlalchemy.sql import func +from app.db.base import Base + + +class UploadHistory(Base): + """ + Upload history tracking + Stores information about uploaded files and their processing status + """ + __tablename__ = "upload_history" + __table_args__ = {'schema': 'fastapi'} + + id = Column(Integer, primary_key=True, index=True) + upload_id = Column(String, unique=True, index=True, nullable=False) + filename = Column(String, nullable=False) + filepath = Column(String, nullable=False) + description = Column(Text) + status = Column(String, default="pending") + job_id = Column(String) + logs = Column(Text) + uploaded_by = Column(String) + uploaded_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updated_at = Column(DateTime(timezone=True), onupdate=func.now()) + + airflow_dag_run_id = Column(String) + airflow_state = Column(String) + processing_started_at = Column(DateTime(timezone=True)) + processing_completed_at = Column(DateTime(timezone=True)) + error_message = Column(Text) diff --git a/03-apiservice/app/models/user.py b/03-apiservice/app/models/user.py new file mode 100644 index 0000000..3e2b999 --- /dev/null +++ b/03-apiservice/app/models/user.py @@ -0,0 +1,55 @@ +""" +User and Role models for local user management +Note: This is separate from Keycloak users - used for tracking and audit +""" +from sqlalchemy import Column, Integer, String, Boolean, DateTime, Table, ForeignKey +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +from app.db.base import Base + +# Association table for many-to-many relationship +user_roles = Table( + 'user_roles', + Base.metadata, + Column('user_id', Integer, ForeignKey('fastapi.users.id'), primary_key=True), + Column('role_id', Integer, ForeignKey('fastapi.roles.id'), primary_key=True), + schema='fastapi' +) + + +class User(Base): + """ + Local user record (synced from Keycloak) + Used for tracking, audit, and local permissions + """ + __tablename__ = "users" + __table_args__ = {'schema': 'fastapi'} + + id = Column(Integer, primary_key=True, index=True) + keycloak_id = Column(String, unique=True, index=True, nullable=False) # Keycloak sub + username = Column(String, unique=True, index=True, nullable=False) + email = Column(String, unique=True, index=True) + full_name = Column(String) + is_active = Column(Boolean, default=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), onupdate=func.now()) + last_login = Column(DateTime(timezone=True)) + + # Relationships + roles = relationship("Role", secondary=user_roles, back_populates="users") + + +class Role(Base): + """ + Roles (synced from Keycloak) + """ + __tablename__ = "roles" + __table_args__ = {'schema': 'fastapi'} + + id = Column(Integer, primary_key=True, index=True) + name = Column(String, unique=True, nullable=False, index=True) # admin, operation + description = Column(String) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + users = relationship("User", secondary=user_roles, back_populates="roles") diff --git a/03-apiservice/app/routes/__init__.py b/03-apiservice/app/routes/__init__.py new file mode 100644 index 0000000..d212dab --- /dev/null +++ b/03-apiservice/app/routes/__init__.py @@ -0,0 +1 @@ +# Routes package diff --git a/03-apiservice/app/routes/admin_users.py b/03-apiservice/app/routes/admin_users.py new file mode 100644 index 0000000..a37a624 --- /dev/null +++ b/03-apiservice/app/routes/admin_users.py @@ -0,0 +1,156 @@ +""" +User and Role management endpoints (Admin only) +Note: This manages local user records synced from Keycloak +API endpoints (/api/v1/*) are NOT affected and continue using API Key authentication +""" +from typing import List +from fastapi import APIRouter, Depends, HTTPException, Request +from sqlalchemy.orm import Session +from pydantic import BaseModel +from datetime import datetime + +from app.db.session import get_db +from app.models.user import User, Role +from app.security.permissions import require_role, Roles +import logging + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/admin/users", tags=["admin-users"]) + + +# Pydantic schemas +class RoleSchema(BaseModel): + id: int + name: str + description: str | None = None + + class Config: + from_attributes = True + + +class UserSchema(BaseModel): + id: int + keycloak_id: str + username: str + email: str | None = None + full_name: str | None = None + is_active: bool + roles: List[RoleSchema] = [] + last_login: datetime | None = None + created_at: datetime + + class Config: + from_attributes = True + + +class UserCreateSchema(BaseModel): + keycloak_id: str + username: str + email: str | None = None + full_name: str | None = None + + +class UserUpdateSchema(BaseModel): + email: str | None = None + full_name: str | None = None + is_active: bool | None = None + role_ids: List[int] | None = None + + +@router.get("/", response_model=List[UserSchema]) +async def list_users( + db: Session = Depends(get_db), + current_user: dict = Depends(require_role(Roles.ADMIN)) +): + """List all users (Admin only)""" + users = db.query(User).all() + return users + + +@router.get("/{user_id}", response_model=UserSchema) +async def get_user( + user_id: int, + db: Session = Depends(get_db), + current_user: dict = Depends(require_role(Roles.ADMIN)) +): + """Get user by ID (Admin only)""" + user = db.query(User).filter(User.id == user_id).first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + return user + + +@router.post("/", response_model=UserSchema) +async def create_user( + user_data: UserCreateSchema, + db: Session = Depends(get_db), + current_user: dict = Depends(require_role(Roles.ADMIN)) +): + """Create new user record (Admin only)""" + # Check if user already exists + existing = db.query(User).filter(User.keycloak_id == user_data.keycloak_id).first() + if existing: + raise HTTPException(status_code=400, detail="User already exists") + + user = User(**user_data.dict()) + db.add(user) + db.commit() + db.refresh(user) + return user + + +@router.put("/{user_id}", response_model=UserSchema) +async def update_user( + user_id: int, + user_data: UserUpdateSchema, + db: Session = Depends(get_db), + current_user: dict = Depends(require_role(Roles.ADMIN)) +): + """Update user (Admin only)""" + user = db.query(User).filter(User.id == user_id).first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + + # Update fields + if user_data.email is not None: + user.email = user_data.email + if user_data.full_name is not None: + user.full_name = user_data.full_name + if user_data.is_active is not None: + user.is_active = user_data.is_active + + # Update roles + if user_data.role_ids is not None: + roles = db.query(Role).filter(Role.id.in_(user_data.role_ids)).all() + user.roles = roles + + db.commit() + db.refresh(user) + return user + + +@router.delete("/{user_id}") +async def delete_user( + user_id: int, + db: Session = Depends(get_db), + current_user: dict = Depends(require_role(Roles.ADMIN)) +): + """Delete user (Admin only)""" + user = db.query(User).filter(User.id == user_id).first() + if not user: + raise HTTPException(status_code=404, detail="User not found") + + db.delete(user) + db.commit() + return {"message": "User deleted successfully"} + + +@router.get("/roles/", response_model=List[RoleSchema]) +async def list_roles( + db: Session = Depends(get_db), + current_user: dict = Depends(require_role(Roles.ADMIN)) +): + """List all roles (Admin only)""" + roles = db.query(Role).all() + return roles diff --git a/03-apiservice/app/routes/auth.py b/03-apiservice/app/routes/auth.py new file mode 100644 index 0000000..02ad5b5 --- /dev/null +++ b/03-apiservice/app/routes/auth.py @@ -0,0 +1,252 @@ +""" +Authentication routes for Keycloak web login +Note: These routes are ONLY for web UI authentication +API endpoints use API Key authentication separately +""" +from fastapi import APIRouter, Request, HTTPException, Query +from fastapi.responses import RedirectResponse +from app.security.keycloak_auth import ( + get_keycloak_client, + get_login_url, + get_logout_url, + get_current_user +) +from app.core.config import settings +import logging + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/auth", tags=["authentication"]) + + +@router.get("/login") +async def login(request: Request, redirect_to: str = Query(default="/")): + """ + Redirect to Keycloak login page + + Args: + redirect_to: Path to redirect after successful login + """ + # Check if already logged in + user = get_current_user(request) + if user: + return RedirectResponse(url=redirect_to) + + # Generate Keycloak login URL + login_url = get_login_url(redirect_to) + return RedirectResponse(url=login_url) + + +@router.get("/callback") +async def auth_callback( + request: Request, + code: str = Query(...), + state: str = Query(default="/") +): + """ + Handle Keycloak callback after login + + Args: + code: Authorization code from Keycloak + state: Original redirect path + """ + try: + if settings.DEBUG_AUTH: + logger.info("=" * 80) + logger.info("AUTHENTICATION CALLBACK RECEIVED") + logger.info(f"Authorization code: {code[:20]}...{code[-20:] if len(code) > 40 else code}") + logger.info(f"State (redirect_to): {state}") + logger.info(f"Request URL: {request.url}") + logger.info(f"Request headers: {dict(request.headers)}") + + keycloak_client = get_keycloak_client() + + if settings.DEBUG_AUTH: + logger.info("-" * 80) + logger.info("EXCHANGING CODE FOR TOKENS") + logger.info(f"Grant type: authorization_code") + logger.info(f"Code: {code[:20]}...") + logger.info(f"Redirect URI: {settings.KEYCLOAK_REDIRECT_URI}") + + # Exchange authorization code for tokens + token_response = keycloak_client.token( + grant_type="authorization_code", + code=code, + redirect_uri=settings.KEYCLOAK_REDIRECT_URI + ) + + if settings.DEBUG_AUTH: + logger.info("TOKEN RESPONSE RECEIVED") + logger.info(f"Access token: {'*' * 20}...{token_response.get('access_token', '')[-20:] if token_response.get('access_token') else 'NONE'}") + logger.info(f"Refresh token: {'Present' if token_response.get('refresh_token') else 'None'}") + logger.info(f"Token type: {token_response.get('token_type', 'N/A')}") + logger.info(f"Expires in: {token_response.get('expires_in', 'N/A')} seconds") + + # Get user information from token + access_token = token_response.get("access_token") + if not access_token: + logger.error("No access token in response!") + if settings.DEBUG_AUTH: + logger.error(f"Full token response: {token_response}") + raise HTTPException( + status_code=400, + detail="No access token received from Keycloak" + ) + + if settings.DEBUG_AUTH: + logger.info("-" * 80) + logger.info("FETCHING USER INFO") + + from jose import jwt + userinfo = jwt.decode(access_token, key="", options={"verify_signature": False, "verify_aud": False}) + logger.info(f"Decoded access_token: {userinfo.get('preferred_username')}") + + # # 2. ดึง id_token ออกมา (ตัวนี้คือหัวใจของ OIDC) + # id_token = token_response.get("id_token") + + # if not id_token: + # logger.error("No id_token in response!") + # if settings.DEBUG_AUTH: + # logger.error(f"Full token response: {token_response}") + # raise HTTPException( + # status_code=400, + # detail="No id_token received from Keycloak" + # ) + + # # 3. Decode id_token เพื่อเอาข้อมูล User (ไม่ต้องใช้ Key เพราะเราเชื่อถือ Connection นี้) + # userinfo = jwt.decode( + # id_token, + # key="", + # options={"verify_signature": False, "verify_aud": False} + # ) + + #userinfo = keycloak_client.userinfo(access_token) + + # Extract roles from token + roles = [] + + # 1. Realm roles (roles ระดับ realm) + if "realm_access" in userinfo and "roles" in userinfo["realm_access"]: + roles.extend(userinfo["realm_access"]["roles"]) + + # 2. Client roles (roles เฉพาะ client apiservice) + if "resource_access" in userinfo and settings.KEYCLOAK_CLIENT_ID in userinfo["resource_access"]: + client_roles = userinfo["resource_access"][settings.KEYCLOAK_CLIENT_ID].get("roles", []) + roles.extend(client_roles) + + # Filter to only include our application roles + user_roles = [r for r in roles if r in ["admin", "operation"]] + + if settings.DEBUG_AUTH: + logger.info("USER INFO RECEIVED") + logger.info(f"Username: {userinfo.get('preferred_username')}") + logger.info(f"Email: {userinfo.get('email')}") + logger.info(f"Name: {userinfo.get('name')}") + logger.info(f"Sub (User ID): {userinfo.get('sub')}") + logger.info(f"All roles from token: {roles}") + logger.info(f"Filtered user roles: {user_roles}") + logger.info(f"Full userinfo keys: {list(userinfo.keys())}") + + # Store user info, roles, and tokens in session + user_session_data = { + "username": userinfo.get("preferred_username"), + "email": userinfo.get("email"), + "name": userinfo.get("name", userinfo.get("preferred_username")), + "sub": userinfo.get("sub"), # User ID + "roles": user_roles, # User roles + "access_token": access_token, + "refresh_token": token_response.get("refresh_token") + } + + request.session["user"] = user_session_data + + if settings.DEBUG_AUTH: + logger.info("-" * 80) + logger.info("SESSION UPDATED") + logger.info(f"Session user data: {dict((k, v) for k, v in user_session_data.items() if k not in ['access_token', 'refresh_token'])}") + + logger.info(f"User {userinfo.get('preferred_username')} logged in successfully") + + # Redirect to original destination + redirect_url = state if state else "/" + + # Ensure redirect URL starts with root_path if set + if settings.ROOT_PATH and not redirect_url.startswith(settings.ROOT_PATH): + redirect_url = f"{settings.ROOT_PATH}{redirect_url}" + + if settings.DEBUG_AUTH: + logger.info(f"Redirecting to: {redirect_url}") + logger.info("=" * 80) + + return RedirectResponse(url=redirect_url, status_code=302) + + except Exception as e: + logger.error(f"Authentication callback failed: {e}") + if settings.DEBUG_AUTH: + import traceback + logger.error("FULL TRACEBACK:") + logger.error(traceback.format_exc()) + logger.error("=" * 80) + raise HTTPException( + status_code=400, + detail=f"Authentication failed: {str(e)}" + ) + + +@router.get("/logout") +async def logout(request: Request): + """ + Logout user and clear session + Redirects to Keycloak logout page + """ + user = get_current_user(request) + + # Clear session + request.session.clear() + + if user: + logger.info(f"User {user.get('username')} logged out") + + # Get Keycloak logout URL + redirect_uri = f"{settings.ROOT_PATH}/" if settings.ROOT_PATH else "/" + logout_url = get_logout_url(redirect_uri) + + return RedirectResponse(url=logout_url) + + +@router.get("/user") +async def get_user_info(request: Request): + """ + Get current authenticated user information + Returns 401 if not authenticated + """ + user = get_current_user(request) + if not user: + raise HTTPException( + status_code=401, + detail="Not authenticated" + ) + + # Return user info without sensitive tokens + return { + "username": user.get("username"), + "email": user.get("email"), + "name": user.get("name"), + "sub": user.get("sub") + } + + +@router.get("/status") +async def auth_status(request: Request): + """ + Check authentication status + Returns whether user is logged in + """ + user = get_current_user(request) + return { + "authenticated": user is not None, + "user": { + "username": user.get("username"), + "name": user.get("name") + } if user else None + } diff --git a/03-apiservice/app/routes/pages.py b/03-apiservice/app/routes/pages.py new file mode 100644 index 0000000..ba35b17 --- /dev/null +++ b/03-apiservice/app/routes/pages.py @@ -0,0 +1,305 @@ +""" +Web page routes for the application +""" +import os +from datetime import datetime +from pathlib import Path +from typing import List + +from fastapi import APIRouter, Request, UploadFile, File, Form, HTTPException, Depends +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates +from pydantic import BaseModel +from sqlalchemy.orm import Session +import asyncio +import logging + +from app.core.config import settings +from app.security.permissions import require_role, Roles +from app.db.session import get_db +from app.models.upload import UploadHistory +from app.services.airflow_client import airflow_client + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Setup templates +templates_dir = Path(__file__).parent.parent / "templates" +templates = Jinja2Templates(directory=str(templates_dir)) + +# Upload directory +UPLOAD_DIR = Path("/data/uploads") +UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + + +class UploadRecordSchema(BaseModel): + id: int + upload_id: str + filename: str + filepath: str + description: str | None = None + status: str + job_id: str | None = None + logs: str | None = None + uploaded_by: str | None = None + uploaded_at: datetime + updated_at: datetime | None = None + + class Config: + from_attributes = True + + +@router.get("/", response_class=HTMLResponse) +async def index(request: Request): + """Landing page with navigation menu""" + user = request.session.get("user") + return templates.TemplateResponse( + "index.html", + { + "request": request, + "root_path": settings.ROOT_PATH, + "user": user + } + ) + + +@router.get("/data-management/finance", response_class=HTMLResponse) +async def finance_page(request: Request): + """Finance Excel upload page - requires operation or admin role""" + user = request.session.get("user") + return templates.TemplateResponse( + "data_management_finance.html", + { + "request": request, + "root_path": settings.ROOT_PATH, + "user": user + } + ) + + +@router.get("/admin/users", response_class=HTMLResponse) +async def admin_users_page( + request: Request, + current_user: dict = Depends(require_role(Roles.ADMIN)) +): + """User management page - Admin only""" + return templates.TemplateResponse( + "admin_users.html", + { + "request": request, + "root_path": settings.ROOT_PATH, + "user": current_user + } + ) + + +@router.post("/data-management/finance/upload") +async def upload_finance_file( + request: Request, + file: UploadFile = File(...), + description: str = Form(None), + db: Session = Depends(get_db) +): + """ + Handle finance Excel file upload + + - Saves file to /data/uploads/ + - Stores upload record in database + - Triggers Airflow job (to be implemented) + - Returns upload record + """ + # Validate file type + if not file.filename.endswith(('.xlsx', '.xls')): + raise HTTPException( + status_code=400, + detail="Invalid file type. Only .xlsx and .xls files are allowed." + ) + + # Generate unique filename + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + safe_filename = file.filename.replace(" ", "_") + unique_filename = f"{timestamp}_{safe_filename}" + filepath = UPLOAD_DIR / unique_filename + + # Save file + try: + content = await file.read() + with open(filepath, "wb") as f: + f.write(content) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to save file: {str(e)}" + ) + + # Get username from session + user = request.session.get("user") + username = user.get("username") if user else "anonymous" + + # Create upload record in database + upload_id = f"upload_{timestamp}" + upload_record = UploadHistory( + upload_id=upload_id, + filename=file.filename, + filepath=str(filepath), + description=description, + status="pending", + uploaded_by=username + ) + + db.add(upload_record) + db.commit() + db.refresh(upload_record) + + # Trigger Airflow DAG with retry logic + airflow_triggered = False + dag_run_id = None + error_msg = None + + max_retries = 3 + retry_delay = 10 # seconds + + for attempt in range(max_retries): + try: + logger.info(f"Triggering Airflow DAG (attempt {attempt + 1}/{max_retries})") + + result = await airflow_client.trigger_finance_dag( + upload_id=upload_id, + filepath=str(filepath), + filename=file.filename, + uploaded_by=username, + description=description + ) + + dag_run_id = result.get("dag_run_id") + airflow_triggered = True + + # Update upload record with Airflow info + upload_record.airflow_dag_run_id = dag_run_id + upload_record.airflow_state = result.get("state", "queued") + upload_record.status = "processing" + db.commit() + + logger.info(f"Airflow DAG triggered successfully: {dag_run_id}") + break + + except Exception as e: + error_msg = str(e) + logger.error(f"Failed to trigger Airflow (attempt {attempt + 1}/{max_retries}): {error_msg}") + + if attempt < max_retries - 1: + logger.info(f"Retrying in {retry_delay} seconds...") + await asyncio.sleep(retry_delay) + else: + logger.error(f"All {max_retries} attempts failed to trigger Airflow") + upload_record.status = "error" + upload_record.error_message = f"Failed to trigger Airflow after {max_retries} attempts: {error_msg}" + db.commit() + + return { + "success": True, + "message": f"File '{file.filename}' uploaded successfully", + "upload_id": upload_id, + "filename": unique_filename, + "airflow_triggered": airflow_triggered, + "dag_run_id": dag_run_id, + "error": error_msg if not airflow_triggered else None + } + + +@router.get("/data-management/finance/uploads") +async def get_uploads(db: Session = Depends(get_db)): + """Get list of all uploads with their status""" + uploads = db.query(UploadHistory).order_by(UploadHistory.uploaded_at.desc()).all() + + # Convert to dict for JSON response + return [ + { + "id": upload.upload_id, + "filename": upload.filename, + "filepath": upload.filepath, + "uploaded_at": upload.uploaded_at.isoformat(), + "description": upload.description, + "status": upload.status, + "job_id": upload.job_id, + "logs": upload.logs, + "uploaded_by": upload.uploaded_by, + "airflow_dag_run_id": upload.airflow_dag_run_id, + "airflow_state": upload.airflow_state, + "processing_started_at": upload.processing_started_at.isoformat() if upload.processing_started_at else None, + "processing_completed_at": upload.processing_completed_at.isoformat() if upload.processing_completed_at else None, + "error_message": upload.error_message + } + for upload in uploads + ] + + +@router.get("/data-management/finance/uploads/{upload_id}") +async def get_upload_status(upload_id: str, db: Session = Depends(get_db)): + """Get status of a specific upload""" + upload = db.query(UploadHistory).filter(UploadHistory.upload_id == upload_id).first() + if not upload: + raise HTTPException(status_code=404, detail="Upload not found") + + return { + "id": upload.upload_id, + "filename": upload.filename, + "filepath": upload.filepath, + "uploaded_at": upload.uploaded_at.isoformat(), + "description": upload.description, + "status": upload.status, + "job_id": upload.job_id, + "logs": upload.logs, + "uploaded_by": upload.uploaded_by, + "airflow_dag_run_id": upload.airflow_dag_run_id, + "airflow_state": upload.airflow_state, + "processing_started_at": upload.processing_started_at.isoformat() if upload.processing_started_at else None, + "processing_completed_at": upload.processing_completed_at.isoformat() if upload.processing_completed_at else None, + "error_message": upload.error_message + } + + +# Placeholder for Airflow integration +async def trigger_airflow_job(filepath: str, upload_id: str) -> str: + """ + Trigger Airflow DAG to process the uploaded file + + Args: + filepath: Path to the uploaded file + upload_id: Unique upload identifier + + Returns: + job_id: Airflow job/run ID + + This function will be implemented when: + - Airflow DAG ID is provided + - Airflow API endpoint is configured + """ + # TODO: Implement Airflow API call + # Example implementation: + # import httpx + # + # airflow_url = "http://airflow-webserver:8080/api/v1/dags/{dag_id}/dagRuns" + # headers = {"Content-Type": "application/json"} + # auth = ("airflow", "airflow") # Use proper credentials + # + # payload = { + # "conf": { + # "filepath": filepath, + # "upload_id": upload_id + # } + # } + # + # async with httpx.AsyncClient() as client: + # response = await client.post( + # airflow_url, + # json=payload, + # headers=headers, + # auth=auth + # ) + # response.raise_for_status() + # result = response.json() + # return result["dag_run_id"] + + raise NotImplementedError("Airflow integration pending DAG ID and endpoint") diff --git a/03-apiservice/app/security/keycloak_auth.py b/03-apiservice/app/security/keycloak_auth.py new file mode 100644 index 0000000..bdbee36 --- /dev/null +++ b/03-apiservice/app/security/keycloak_auth.py @@ -0,0 +1,146 @@ +""" +Keycloak authentication for web pages +Note: This is ONLY for web UI authentication, NOT for API endpoints +API endpoints use API Key authentication (see app/security/dependencies.py) +""" +from typing import Optional +from fastapi import HTTPException, Request, status +from keycloak import KeycloakOpenID +from app.core.config import settings +import logging + +logger = logging.getLogger(__name__) + +# Initialize Keycloak OpenID client +def get_keycloak_client() -> KeycloakOpenID: + """Get Keycloak OpenID client instance""" + try: + if settings.DEBUG_AUTH: + logger.info("=" * 80) + logger.info("KEYCLOAK CLIENT INITIALIZATION") + logger.info(f"Server URL: {settings.KEYCLOAK_SERVER_URL}") + logger.info(f"Realm: {settings.KEYCLOAK_REALM}") + logger.info(f"Client ID: {settings.KEYCLOAK_CLIENT_ID}") + logger.info(f"Client Secret: {'*' * len(settings.KEYCLOAK_CLIENT_SECRET) if settings.KEYCLOAK_CLIENT_SECRET else 'NOT SET'}") + logger.info(f"Redirect URI: {settings.KEYCLOAK_REDIRECT_URI}") + logger.info("=" * 80) + + return KeycloakOpenID( + server_url=settings.KEYCLOAK_SERVER_URL, + client_id=settings.KEYCLOAK_CLIENT_ID, + realm_name=settings.KEYCLOAK_REALM, + client_secret_key=settings.KEYCLOAK_CLIENT_SECRET, + verify=True + ) + except Exception as e: + logger.error(f"Failed to initialize Keycloak client: {e}") + if settings.DEBUG_AUTH: + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + raise + + +def get_current_user(request: Request) -> Optional[dict]: + """ + Get current authenticated user from session + Returns None if not authenticated + """ + return request.session.get("user") + + +def require_user(request: Request) -> dict: + """ + Dependency to require authenticated user + Raises 401 if not authenticated + """ + user = get_current_user(request) + if not user: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Authentication required" + ) + return user + + +def get_login_url(redirect_to: str = "/") -> str: + """ + Generate Keycloak login URL + + Args: + redirect_to: Path to redirect after successful login + + Returns: + Keycloak authorization URL + """ + try: + if settings.DEBUG_AUTH: + logger.info("=" * 80) + logger.info("GENERATING LOGIN URL") + logger.info(f"Redirect to after login: {redirect_to}") + logger.info(f"Keycloak redirect URI: {settings.KEYCLOAK_REDIRECT_URI}") + + keycloak_client = get_keycloak_client() + auth_url = keycloak_client.auth_url( + redirect_uri=settings.KEYCLOAK_REDIRECT_URI, + state=redirect_to + ) + + if settings.DEBUG_AUTH: + logger.info(f"Generated auth URL: {auth_url}") + logger.info("=" * 80) + + return auth_url + except Exception as e: + logger.error(f"Failed to generate login URL: {e}") + if settings.DEBUG_AUTH: + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Authentication service unavailable" + ) + + +async def verify_token(token: str) -> dict: + """ + Verify and decode Keycloak access token + + Args: + token: Access token from Keycloak + + Returns: + User information from token + """ + try: + keycloak_client = get_keycloak_client() + userinfo = keycloak_client.userinfo(token) + return userinfo + except Exception as e: + logger.error(f"Token verification failed: {e}") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired token" + ) + + +def get_logout_url(redirect_uri: str = None) -> str: + """ + Generate Keycloak logout URL + + Args: + redirect_uri: Where to redirect after logout + + Returns: + Keycloak logout URL + """ + try: + keycloak_client = get_keycloak_client() + if redirect_uri is None: + redirect_uri = f"{settings.ROOT_PATH}/" if settings.ROOT_PATH else "/" + + logout_url = keycloak_client.logout_url(redirect_uri=redirect_uri) + return logout_url + except Exception as e: + logger.error(f"Failed to generate logout URL: {e}") + # Return simple redirect if Keycloak logout fails + return redirect_uri or "/" diff --git a/03-apiservice/app/security/permissions.py b/03-apiservice/app/security/permissions.py new file mode 100644 index 0000000..12e40eb --- /dev/null +++ b/03-apiservice/app/security/permissions.py @@ -0,0 +1,151 @@ +""" +Role-based permission system for web pages +Note: API endpoints (/api/v1/*) use API Key authentication and are not affected by this +""" +from typing import List +from fastapi import HTTPException, Request, status +from app.core.config import settings +import logging + +logger = logging.getLogger(__name__) + + +class Roles: + """Role constants""" + ADMIN = "admin" + OPERATION = "operation" + + +def get_user_roles(request: Request) -> List[str]: + """ + Get roles from session + + Args: + request: FastAPI request object + + Returns: + List of role names + """ + user = request.session.get("user") + if not user: + return [] + return user.get("roles", []) + + +def has_role(request: Request, required_role: str) -> bool: + """ + Check if user has specific role + + Args: + request: FastAPI request object + required_role: Role name to check + + Returns: + True if user has the role, False otherwise + """ + user_roles = get_user_roles(request) + return required_role in user_roles + + +def has_any_role(request: Request, required_roles: List[str]) -> bool: + """ + Check if user has any of the required roles + + Args: + request: FastAPI request object + required_roles: List of role names + + Returns: + True if user has at least one of the roles, False otherwise + """ + user_roles = get_user_roles(request) + return any(role in user_roles for role in required_roles) + + +def require_role(required_role: str): + """ + Dependency to require specific role + + Args: + required_role: Role name required to access the endpoint + + Returns: + Dependency function that checks for the role + + Raises: + HTTPException: 401 if not authenticated, 403 if missing required role + """ + def check_role(request: Request): + user = request.session.get("user") + if not user: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Authentication required" + ) + + user_roles = user.get("roles", []) + + if required_role not in user_roles: + logger.warning( + f"Access denied: User {user.get('username')} " + f"(roles: {user_roles}) tried to access resource requiring '{required_role}'" + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Access denied. Required role: {required_role}" + ) + + return user + + return check_role + + +def require_any_role(required_roles: List[str]): + """ + Dependency to require any of the specified roles + + Args: + required_roles: List of role names, user needs at least one + + Returns: + Dependency function that checks for roles + + Raises: + HTTPException: 401 if not authenticated, 403 if missing all required roles + """ + def check_roles(request: Request): + user = request.session.get("user") + if not user: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Authentication required" + ) + + user_roles = user.get("roles", []) + + if not any(role in user_roles for role in required_roles): + logger.warning( + f"Access denied: User {user.get('username')} " + f"(roles: {user_roles}) tried to access resource requiring one of: {required_roles}" + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Access denied. Required roles: {', '.join(required_roles)}" + ) + + return user + + return check_roles + + +def is_admin(request: Request) -> bool: + """ + Check if user is admin + + Args: + request: FastAPI request object + + Returns: + True if user has admin role, False otherwise + """ + return has_role(request, Roles.ADMIN) diff --git a/03-apiservice/app/services/airflow_client.py b/03-apiservice/app/services/airflow_client.py new file mode 100644 index 0000000..24f7266 --- /dev/null +++ b/03-apiservice/app/services/airflow_client.py @@ -0,0 +1,152 @@ +""" +Airflow API Client for triggering DAGs +""" +import httpx +import logging +from typing import Dict, Any, Optional +from datetime import datetime + +from app.core.config import settings + +logger = logging.getLogger(__name__) + + +class AirflowClient: + """Client for interacting with Airflow REST API""" + + def __init__(self): + self.api_url = settings.AIRFLOW_API_URL.rstrip('/') + self.api_token = settings.AIRFLOW_API_TOKEN + self.headers = { + "Content-Type": "application/json", + "Accept": "application/json" + } + + if self.api_token: + self.headers["Authorization"] = f"Bearer {self.api_token}" + else: + logger.warning("AIRFLOW_API_TOKEN not set - API calls may fail") + + async def trigger_dag( + self, + dag_id: str, + conf: Dict[str, Any], + logical_date: Optional[str] = None + ) -> Dict[str, Any]: + """ + Trigger an Airflow DAG run + + Args: + dag_id: DAG identifier + conf: Configuration to pass to the DAG (JSON serializable dict) + logical_date: Optional logical date for the DAG run (ISO format) + + Returns: + Dict containing dag_run_id and other metadata + + Raises: + httpx.HTTPError: If the API call fails + """ + url = f"{self.api_url}/api/v1/dags/{dag_id}/dagRuns" + + payload = { + "conf": conf + } + + if logical_date: + payload["logical_date"] = logical_date + else: + payload["logical_date"] = datetime.utcnow().isoformat() + "Z" + + logger.info(f"Triggering Airflow DAG: {dag_id}") + logger.debug(f"Payload: {payload}") + + try: + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.post( + url, + json=payload, + headers=self.headers + ) + + response.raise_for_status() + result = response.json() + + logger.info(f"DAG triggered successfully: {result.get('dag_run_id')}") + return result + + except httpx.HTTPStatusError as e: + logger.error(f"HTTP error triggering DAG {dag_id}: {e.response.status_code} - {e.response.text}") + raise + except httpx.RequestError as e: + logger.error(f"Request error triggering DAG {dag_id}: {str(e)}") + raise + except Exception as e: + logger.error(f"Unexpected error triggering DAG {dag_id}: {str(e)}") + raise + + async def get_dag_run_status( + self, + dag_id: str, + dag_run_id: str + ) -> Dict[str, Any]: + """ + Get the status of a DAG run + + Args: + dag_id: DAG identifier + dag_run_id: DAG run identifier + + Returns: + Dict containing DAG run status and metadata + """ + url = f"{self.api_url}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}" + + try: + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.get(url, headers=self.headers) + response.raise_for_status() + return response.json() + + except httpx.HTTPError as e: + logger.error(f"Error getting DAG run status: {str(e)}") + raise + + async def trigger_finance_dag( + self, + upload_id: str, + filepath: str, + filename: str, + uploaded_by: str, + description: Optional[str] = None + ) -> Dict[str, Any]: + """ + Trigger the finance Excel processing DAG + + Args: + upload_id: Unique upload identifier + filepath: Full path to the uploaded file + filename: Original filename + uploaded_by: Username who uploaded the file + description: Optional description + + Returns: + Dict containing dag_run_id and other metadata + """ + conf = { + "upload_id": upload_id, + "filepath": filepath, + "filename": filename, + "uploaded_by": uploaded_by, + "description": description or "", + "triggered_at": datetime.utcnow().isoformat() + } + + return await self.trigger_dag( + dag_id=settings.AIRFLOW_DAG_ID_FINANCE, + conf=conf + ) + + +# Singleton instance +airflow_client = AirflowClient() diff --git a/03-apiservice/app/templates/admin_users.html b/03-apiservice/app/templates/admin_users.html new file mode 100644 index 0000000..1142653 --- /dev/null +++ b/03-apiservice/app/templates/admin_users.html @@ -0,0 +1,360 @@ + + + + + + User Management - Admin + + + +
+ ← Back to Dashboard + +
+

👥 User Management

+ {% if user %} + + {% endif %} +
+ +
+ +
+
+
-
+
Total Users
+
+
+
-
+
Active Users
+
+
+
-
+
Admins
+
+
+
-
+
Operations
+
+
+ +
+
Loading users...
+
+
+ + + + diff --git a/03-apiservice/app/templates/data_management_finance.html b/03-apiservice/app/templates/data_management_finance.html new file mode 100644 index 0000000..c298610 --- /dev/null +++ b/03-apiservice/app/templates/data_management_finance.html @@ -0,0 +1,635 @@ + + + + + + Finance Excel Upload - Sriphat Data Platform + + + +
+ ← Back to Dashboard + +
+

💰 Finance Excel Upload

+

Upload Excel files for financial data processing

+ + {% if user %} + + {% endif %} +
+ +
+ +
+

📤 Upload File

+
+
+ +
+ + +
+
+
+ +
+ + +
+ + +
+
+ +
+

📋 Upload History

+
+
+
📭
+

No uploads yet

+
+
+
+
+ + + + diff --git a/03-apiservice/app/templates/index.html b/03-apiservice/app/templates/index.html new file mode 100644 index 0000000..aa7b1f5 --- /dev/null +++ b/03-apiservice/app/templates/index.html @@ -0,0 +1,296 @@ + + + + + + Sriphat Data Platform + + + +
+
+

🏥 Sriphat Data Platform

+

Integrated Data Management & Analytics Platform

+ + {% if user %} + + {% endif %} +
+ +
+ + + 🗄️ +

Supabase

+

PostgreSQL database with real-time capabilities and REST API

+ + + + + 📚 +

API Documentation

+

Interactive API documentation and testing interface

+ + + + + 🔄 +

Airflow

+

Workflow orchestration and data pipeline management

+ + + + + 🔌 +

Airbyte

+

Data integration and ETL platform

+ + + +
+ 📊 +

Data Management ▼

+

Upload and manage data files

+
+ + 💰 Finance Excel Upload + +
+
+ + + + 🔧 +

DBT

+

Data transformation and modeling

+ + + + + 📈 +

Superset

+

Business intelligence and data visualization

+ + + + {% if user and user.roles and 'admin' in user.roles %} + + 👥 +

User Management

+

Manage users and roles (Admin only)

+ + {% endif %} +
+
+ + + + diff --git a/03-apiservice/app/utils/supabase_client.py b/03-apiservice/app/utils/supabase_client.py index 7f69658..7d0ee16 100644 --- a/03-apiservice/app/utils/supabase_client.py +++ b/03-apiservice/app/utils/supabase_client.py @@ -48,7 +48,7 @@ async def upsert_to_supabase( headers["Prefer"] += f",on_conflict={on_conflict}" try: - async with httpx.AsyncClient(timeout=30.0) as client: + async with httpx.AsyncClient(timeout=30.0, verify=False) as client: response = await client.post(url, json=data, headers=headers) response.raise_for_status() return { @@ -102,7 +102,7 @@ def upsert_to_supabase_sync( headers["Prefer"] += f",on_conflict={on_conflict}" try: - with httpx.Client(timeout=30.0) as client: + with httpx.Client(timeout=30.0, verify=False) as client: response = client.post(url, json=data, headers=headers) response.raise_for_status() return { diff --git a/03-apiservice/docker-compose.yml b/03-apiservice/docker-compose.yml index 1ab1219..4998942 100644 --- a/03-apiservice/docker-compose.yml +++ b/03-apiservice/docker-compose.yml @@ -1,6 +1,12 @@ +x-common-configs: &common-config + extra_hosts: + - "dev.sriphat.com:192.168.100.9" + pull_policy: ${DOCKER_PULL_POLICY:-missing} + services: apiservice: - build: . + #build: . + image: 03-apiservice-apiservice:latest container_name: apiservice env_file: - .env @@ -17,6 +23,13 @@ services: - ADMIN_SECRET_KEY=${ADMIN_SECRET_KEY} - ADMIN_USERNAME=${ADMIN_USERNAME} - ADMIN_PASSWORD=${ADMIN_PASSWORD} + - API_KEY_ENC_SECRET=${ADMIN_SECRET_KEY} + - DEBUG_AUTH=${DEBUG_AUTH:-false} + - KEYCLOAK_SERVER_URL=${KEYCLOAK_SERVER_URL} + - KEYCLOAK_REALM=${KEYCLOAK_REALM} + - KEYCLOAK_CLIENT_ID=${API_KEYCLOAK_CLIENT_ID} + - KEYCLOAK_CLIENT_SECRET=${API_KEYCLOAK_CLIENT_SECRET} + - KEYCLOAK_REDIRECT_URI=${API_KEYCLOAK_REDIRECT_URI} - LOG_LEVEL=debug ports: - "8040:8040" @@ -25,6 +38,7 @@ services: volumes: - ./app:/app/app - .env:/app/.env + - ./data/uploads:/data/uploads restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8040/apiservice/docs"] @@ -32,6 +46,9 @@ services: timeout: 10s retries: 3 start_period: 40s + # extra_hosts: + # - "dev.sriphat.com:192.168.100.9" + <<: *common-config networks: shared_data_network: diff --git a/03-apiservice/requirements.txt b/03-apiservice/requirements.txt index 58c0093..615ef82 100644 --- a/03-apiservice/requirements.txt +++ b/03-apiservice/requirements.txt @@ -14,4 +14,7 @@ httpx==0.28.1 WTForms #==3.2.1 cryptography==42.0.5 +python-keycloak==3.9.0 +Authlib==1.3.0 +python-jose[cryptography]==3.3.0 diff --git a/06-analytics/Dockerfile.nginx b/06-analytics/Dockerfile.nginx new file mode 100644 index 0000000..b823d27 --- /dev/null +++ b/06-analytics/Dockerfile.nginx @@ -0,0 +1,46 @@ +FROM apache/superset:latest + +USER root + +ENV PATH="/app/.venv/bin:$PATH" + +RUN python -m ensurepip --upgrade && /app/.venv/bin/python -m pip install --upgrade pip setuptools wheel + +RUN /app/.venv/bin/pip install --no-cache-dir psycopg2-binary + +# Install nginx and supervisor +RUN apt-get update && apt-get install -y --no-install-recommends nginx supervisor && rm -rf /var/lib/apt/lists/* + +# Copy nginx config +COPY nginx-superset.conf /etc/nginx/sites-available/default + +# Create supervisor config to run both nginx and gunicorn +RUN cat > /etc/supervisor/conf.d/supervisord.conf << 'EOF' +[supervisord] +nodaemon=true +user=root + +[program:nginx] +command=nginx -g "daemon off;" +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:superset] +command=/app/.venv/bin/gunicorn --bind 127.0.0.1:8088 --workers 4 --timeout 120 --limit-request-line 0 --limit-request-field_size 0 "superset.app:create_app()" +directory=/app +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +user=superset +EOF + +EXPOSE 80 + +USER superset diff --git a/06-analytics/nginx-superset.conf b/06-analytics/nginx-superset.conf new file mode 100644 index 0000000..07857d1 --- /dev/null +++ b/06-analytics/nginx-superset.conf @@ -0,0 +1,61 @@ +server { + listen 80; + server_name localhost; + + # Handle API endpoints directly - no rewrite needed + location /superset/api/ { + proxy_pass http://superset:8088/api/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Accept-Encoding ""; + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + } + + # Strip /superset/ prefix, proxy to Superset at root + location /superset/ { + proxy_pass http://superset:8088/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Accept-Encoding ""; + proxy_read_timeout 300s; + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + + # Rewrite redirect Location headers to add /superset prefix + proxy_redirect ~^(https?://[^/]+)?/(.*)$ $1/superset/$2; + + # Rewrite URLs in HTML responses + sub_filter_once off; + sub_filter_types text/html application/json; + sub_filter 'href="/' 'href="/superset/'; + sub_filter 'src="/' 'src="/superset/'; + sub_filter 'action="/' 'action="/superset/'; + sub_filter '"/static/' '"/superset/static/'; + sub_filter '"/api/' '"/superset/api/'; + sub_filter '"/superset/superset/' '"/superset/superset/'; + sub_filter '"/login/' '"/superset/login/'; + sub_filter '"/logout/' '"/superset/logout/'; + sub_filter '"/lang/' '"/superset/lang/'; + } + + location = /superset { + return 301 /superset/; + } + + # Health check for Docker + location = /health { + proxy_pass http://superset:8088/health; + } +} diff --git a/07-minio/.env.example b/07-minio/.env.example new file mode 100644 index 0000000..5634656 --- /dev/null +++ b/07-minio/.env.example @@ -0,0 +1,55 @@ +# MinIO Configuration +# Copy this file to .env and update with your values + +# ============================================================================ +# MinIO Credentials +# ============================================================================ +MINIO_ROOT_USER=minioadmin +MINIO_ROOT_PASSWORD=minioadmin_secure_password_2026 + +# ============================================================================ +# MinIO Ports +# ============================================================================ +MINIO_API_PORT=9000 +MINIO_CONSOLE_PORT=9001 + +# ============================================================================ +# MinIO Server URLs (for reverse proxy) +# ============================================================================ +# API endpoint URL (for S3 API access) +MINIO_SERVER_URL=https://ai.sriphat.com/minio + +# Console UI URL (for web interface) +MINIO_BROWSER_REDIRECT_URL=https://ai.sriphat.com/minio-console + +# ============================================================================ +# MinIO Region +# ============================================================================ +MINIO_REGION=ap-southeast-1 + +# ============================================================================ +# Keycloak Integration (OpenID Connect) +# ============================================================================ +# Keycloak OpenID configuration URL +# Format: https://{keycloak-domain}/realms/{realm-name}/.well-known/openid-configuration +MINIO_IDENTITY_OPENID_CONFIG_URL=https://ai.sriphat.com/keycloak/realms/sriphat/.well-known/openid-configuration + +# MinIO client in Keycloak +MINIO_IDENTITY_OPENID_CLIENT_ID=minio + +# Client secret from Keycloak +MINIO_IDENTITY_OPENID_CLIENT_SECRET=your-minio-client-secret-here + +# Claim name for policy mapping (default: policy) +MINIO_IDENTITY_OPENID_CLAIM_NAME=policy + +# OpenID scopes +MINIO_IDENTITY_OPENID_SCOPES=openid,profile,email + +# Redirect URI after authentication +MINIO_IDENTITY_OPENID_REDIRECT_URI=https://ai.sriphat.com/minio-console/oauth_callback + +# ============================================================================ +# Timezone +# ============================================================================ +TZ=Asia/Bangkok diff --git a/07-minio/.gitignore b/07-minio/.gitignore new file mode 100644 index 0000000..3cbe5a1 --- /dev/null +++ b/07-minio/.gitignore @@ -0,0 +1,30 @@ +# Environment variables +.env + +# Data directory (persistent storage) +data/ + +# SSL certificates +certs/ + +# Logs +*.log + +# Backup files +*.tar.gz +*.zip + +# Temporary files +*.tmp +*.temp + +# OS files +.DS_Store +Thumbs.db + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo +*~ diff --git a/07-minio/KEYCLOAK_INTEGRATION.md b/07-minio/KEYCLOAK_INTEGRATION.md new file mode 100644 index 0000000..7aebf87 --- /dev/null +++ b/07-minio/KEYCLOAK_INTEGRATION.md @@ -0,0 +1,362 @@ +# MinIO Keycloak Integration Guide + +Complete guide for integrating MinIO with Keycloak for SSO authentication. + +## 🎯 Overview + +MinIO supports OpenID Connect (OIDC) authentication, allowing users to log in to MinIO Console using Keycloak credentials. This integration provides: + +- **Single Sign-On (SSO)** - Users authenticate once with Keycloak +- **Centralized User Management** - Manage users in Keycloak +- **Role-Based Access Control** - Map Keycloak roles to MinIO policies +- **Secure Authentication** - OAuth 2.0 / OpenID Connect flow + +## 📋 Prerequisites + +- Keycloak instance running and accessible +- MinIO instance running +- Admin access to both Keycloak and MinIO + +## 🔧 Setup Steps + +### **Step 1: Create MinIO Client in Keycloak** + +1. **Login to Keycloak Admin Console** + ``` + https://ai.sriphat.com/keycloak + ``` + +2. **Select Realm** + - Go to your realm (e.g., `sriphat`) + +3. **Create Client** + - Navigate to: **Clients** → **Create Client** + - **Client ID**: `minio` + - **Client Type**: `OpenID Connect` + - **Client Protocol**: `openid-connect` + - Click **Next** + +4. **Capability Config** + - **Client authentication**: `ON` + - **Authorization**: `OFF` + - **Authentication flow**: + - ✅ Standard flow + - ✅ Direct access grants + - ❌ Implicit flow + - ❌ Service accounts roles + - Click **Next** + +5. **Login Settings** + - **Root URL**: `https://ai.sriphat.com/minio-console` + - **Home URL**: `https://ai.sriphat.com/minio-console` + - **Valid redirect URIs**: + ``` + https://ai.sriphat.com/minio-console/* + https://ai.sriphat.com/minio-console/oauth_callback + ``` + - **Valid post logout redirect URIs**: `https://ai.sriphat.com/minio-console` + - **Web origins**: `https://ai.sriphat.com` + - Click **Save** + +6. **Get Client Secret** + - Go to **Credentials** tab + - Copy the **Client Secret** + - Save this for `.env` configuration + +### **Step 2: Create Client Scope for MinIO Policy** + +1. **Create Client Scope** + - Navigate to: **Client Scopes** → **Create client scope** + - **Name**: `minio-authorization` + - **Type**: `Optional` + - **Protocol**: `OpenID Connect` + - **Display on consent screen**: `OFF` + - Click **Save** + +2. **Add Mapper for Policy Claim** + - Go to **Mappers** tab + - Click **Add mapper** → **By configuration** + - Select **User Attribute** + - **Name**: `minio-policy` + - **User Attribute**: `minio_policy` + - **Token Claim Name**: `policy` + - **Claim JSON Type**: `String` + - **Add to ID token**: `ON` + - **Add to access token**: `ON` + - **Add to userinfo**: `ON` + - Click **Save** + +3. **Assign Scope to MinIO Client** + - Go to **Clients** → `minio` + - Go to **Client scopes** tab + - Click **Add client scope** + - Select `minio-authorization` + - Choose **Optional** + - Click **Add** + +### **Step 3: Create MinIO Policies in Keycloak** + +MinIO uses policies to control access. Common policies: + +- `consoleAdmin` - Full admin access +- `readonly` - Read-only access +- `readwrite` - Read and write access +- `diagnostics` - Diagnostics access + +**Add Policy to Users:** + +1. **Go to Users** + - Navigate to: **Users** → Select user + +2. **Add Attribute** + - Go to **Attributes** tab + - Click **Add attribute** + - **Key**: `minio_policy` + - **Value**: `consoleAdmin` (or other policy) + - Click **Save** + +### **Step 4: Configure MinIO Environment Variables** + +Update `07-minio/.env`: + +```bash +# Keycloak Integration +MINIO_IDENTITY_OPENID_CONFIG_URL=https://ai.sriphat.com/keycloak/realms/sriphat/.well-known/openid-configuration +MINIO_IDENTITY_OPENID_CLIENT_ID=minio +MINIO_IDENTITY_OPENID_CLIENT_SECRET=your-client-secret-from-step-1 +MINIO_IDENTITY_OPENID_CLAIM_NAME=policy +MINIO_IDENTITY_OPENID_SCOPES=openid,profile,email,minio-authorization +MINIO_IDENTITY_OPENID_REDIRECT_URI=https://ai.sriphat.com/minio-console/oauth_callback +``` + +### **Step 5: Restart MinIO** + +```bash +cd 07-minio +docker compose down +docker compose up -d +``` + +### **Step 6: Test Authentication** + +1. **Access MinIO Console** + ``` + https://ai.sriphat.com/minio-console + ``` + +2. **Click "Login with SSO"** + - You'll be redirected to Keycloak + - Login with Keycloak credentials + - After successful authentication, you'll be redirected back to MinIO Console + +## 🔐 MinIO Policies + +### **Default Policies** + +MinIO comes with built-in policies: + +| Policy | Description | +|--------|-------------| +| `consoleAdmin` | Full admin access to console and buckets | +| `readonly` | Read-only access to buckets | +| `readwrite` | Read and write access to buckets | +| `diagnostics` | Access to diagnostics and monitoring | +| `writeonly` | Write-only access (upload only) | + +### **Custom Policies** + +Create custom policies in MinIO Console or via `mc` CLI: + +```bash +# Install mc (MinIO Client) +wget https://dl.min.io/client/mc/release/linux-amd64/mc +chmod +x mc +sudo mv mc /usr/local/bin/ + +# Configure mc +mc alias set myminio https://ai.sriphat.com/minio minioadmin minioadmin_secure_password_2026 + +# Create custom policy +cat > custom-policy.json < minio.log +``` + +### **Metrics** + +```bash +# View server info +mc admin info myminio + +# View server stats +mc admin prometheus metrics myminio +``` + +### **Disk Usage** + +```bash +# Check disk usage +mc admin info myminio + +# Check bucket size +mc du myminio/my-bucket +``` + +## 🐛 Troubleshooting + +### **Issue: Cannot access MinIO Console** + +**Check:** +```bash +# Verify container is running +docker ps | grep minio + +# Check logs +docker logs minio + +# Test direct access +curl http://192.168.100.9:9001 +``` + +**Solution:** +- Ensure container is running: `docker compose up -d` +- Check firewall rules +- Verify Nginx configuration + +### **Issue: SSO login not working** + +**Check:** +```bash +# Verify Keycloak config +docker exec minio printenv | grep MINIO_IDENTITY_OPENID + +# Test Keycloak connectivity +docker exec minio curl -k https://ai.sriphat.com/keycloak/realms/sriphat/.well-known/openid-configuration +``` + +**Solution:** +- Verify all Keycloak environment variables are set +- Check client secret is correct +- Ensure redirect URIs match in Keycloak +- See `KEYCLOAK_INTEGRATION.md` for detailed troubleshooting + +### **Issue: Upload fails** + +**Check:** +```bash +# Check disk space +df -h + +# Check permissions +ls -la data/ +``` + +**Solution:** +- Ensure sufficient disk space +- Check directory permissions: `chmod 755 data/` +- Increase `client_max_body_size` in Nginx + +### **Issue: S3 API connection refused** + +**Check:** +```bash +# Test API endpoint +curl -k https://ai.sriphat.com/minio/ + +# Test direct connection +curl http://192.168.100.9:9000/ +``` + +**Solution:** +- Verify `MINIO_SERVER_URL` is set correctly +- Check Nginx proxy configuration +- Ensure port 9000 is accessible + +## 🔄 Maintenance + +### **Backup** + +```bash +# Backup data directory +tar -czf minio-backup-$(date +%Y%m%d).tar.gz data/ + +# Backup to remote location +rsync -avz data/ user@backup-server:/backups/minio/ +``` + +### **Update MinIO** + +```bash +# Pull latest image +docker compose pull + +# Restart with new image +docker compose up -d + +# Verify version +docker exec minio minio --version +``` + +### **Restore** + +```bash +# Stop MinIO +docker compose down + +# Restore data +tar -xzf minio-backup-20260325.tar.gz + +# Start MinIO +docker compose up -d +``` + +## 📚 Documentation + +- **MinIO Official Docs**: https://min.io/docs/minio/linux/ +- **S3 API Reference**: https://docs.aws.amazon.com/AmazonS3/latest/API/ +- **Keycloak Integration**: See `KEYCLOAK_INTEGRATION.md` +- **Nginx Configuration**: See `nginx-minio.conf` + +## 🎯 Use Cases + +### **1. Data Lake Storage** +- Store raw data files (CSV, JSON, Parquet) +- Integrate with Spark, Pandas, Dask +- Version control for datasets + +### **2. Backup Storage** +- Database backups +- Application backups +- Log archival + +### **3. Media Storage** +- Images, videos, documents +- CDN integration +- Static website hosting + +### **4. ML/AI Workflows** +- Model storage +- Training data storage +- Experiment artifacts + +### **5. Application Storage** +- User uploads +- Generated reports +- Temporary files + +## 🎉 Summary + +**What You Have:** +- ✅ MinIO object storage service +- ✅ Persistent storage with volume mounts +- ✅ HTTPS access via Nginx reverse proxy +- ✅ Keycloak SSO integration ready +- ✅ S3-compatible API +- ✅ Web console for management +- ✅ Health checks and monitoring + +**Access:** +- Console: `https://ai.sriphat.com/minio-console` +- API: `https://ai.sriphat.com/minio` + +**Next Steps:** +1. Configure `.env` file +2. Start MinIO: `docker compose up -d` +3. Setup Keycloak integration (optional) +4. Configure Nginx reverse proxy +5. Create buckets and start using! + +For detailed Keycloak SSO setup, see `KEYCLOAK_INTEGRATION.md` 🚀 diff --git a/07-minio/docker-compose.yml b/07-minio/docker-compose.yml new file mode 100644 index 0000000..fdb5299 --- /dev/null +++ b/07-minio/docker-compose.yml @@ -0,0 +1,50 @@ +version: '3.8' + +services: + minio: + image: minio/minio:latest + container_name: minio + command: server /data --console-address ":9001" + ports: + - "${MINIO_API_PORT:-9000}:9000" + - "${MINIO_CONSOLE_PORT:-9001}:9001" + environment: + # MinIO credentials + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + + # Server settings + MINIO_SERVER_URL: ${MINIO_SERVER_URL:-https://ai.sriphat.com/minio} + MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL:-https://ai.sriphat.com/minio-console} + + # Region + MINIO_REGION: ${MINIO_REGION:-ap-southeast-1} + + # Identity OpenID (Keycloak) + MINIO_IDENTITY_OPENID_CONFIG_URL: ${MINIO_IDENTITY_OPENID_CONFIG_URL} + MINIO_IDENTITY_OPENID_CLIENT_ID: ${MINIO_IDENTITY_OPENID_CLIENT_ID} + MINIO_IDENTITY_OPENID_CLIENT_SECRET: ${MINIO_IDENTITY_OPENID_CLIENT_SECRET} + MINIO_IDENTITY_OPENID_CLAIM_NAME: ${MINIO_IDENTITY_OPENID_CLAIM_NAME:-policy} + MINIO_IDENTITY_OPENID_SCOPES: ${MINIO_IDENTITY_OPENID_SCOPES:-openid,profile,email} + MINIO_IDENTITY_OPENID_REDIRECT_URI: ${MINIO_IDENTITY_OPENID_REDIRECT_URI} + + # Timezone + TZ: ${TZ:-Asia/Bangkok} + volumes: + # Persistent storage + - ./data:/data + # SSL certificates (if using direct HTTPS) + - ./certs:/root/.minio/certs:ro + networks: + - shared_data_network + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + +networks: + shared_data_network: + external: true diff --git a/07-minio/nginx-minio.conf b/07-minio/nginx-minio.conf new file mode 100644 index 0000000..5edc9c4 --- /dev/null +++ b/07-minio/nginx-minio.conf @@ -0,0 +1,104 @@ +# MinIO Nginx Configuration +# For use with Nginx Proxy Manager or standalone Nginx +# This configuration provides HTTPS access to MinIO API and Console + +# ============================================================================ +# MinIO S3 API - Port 9000 +# Subpath: /minio +# ============================================================================ +location /minio/ { + # Rewrite path to remove /minio prefix + rewrite ^/minio/(.*) /$1 break; + + # Forward to MinIO API + proxy_pass http://192.168.100.9:9000; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # Required for MinIO + proxy_set_header X-NginX-Proxy true; + + # Disable buffering for large uploads + proxy_buffering off; + proxy_request_buffering off; + + # Timeouts for large file uploads/downloads + proxy_connect_timeout 300; + proxy_send_timeout 300; + proxy_read_timeout 300; + send_timeout 300; + + # Max upload size (adjust as needed) + client_max_body_size 0; +} + +# ============================================================================ +# MinIO Console (Web UI) - Port 9001 +# Subpath: /minio-console +# ============================================================================ +location /minio-console/ { + # Forward to MinIO Console + proxy_pass http://192.168.100.9:9001/; + + # Preserve headers + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + + # WebSocket support for real-time updates + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Disable buffering + proxy_buffering off; + proxy_request_buffering off; + + # Timeouts + proxy_connect_timeout 300; + proxy_send_timeout 300; + proxy_read_timeout 300; +} + +# ============================================================================ +# MinIO Console Assets +# ============================================================================ +location /minio-console/assets/ { + proxy_pass http://192.168.100.9:9001/assets/; + proxy_set_header Host $host; + proxy_cache_valid 200 1d; + add_header Cache-Control "public, immutable"; +} + +# ============================================================================ +# MinIO Console API +# ============================================================================ +location /minio-console/api/ { + proxy_pass http://192.168.100.9:9001/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_buffering off; +} + +# ============================================================================ +# MinIO Health Check +# ============================================================================ +location /minio/health { + proxy_pass http://192.168.100.9:9000/minio/health; + proxy_set_header Host $host; + access_log off; +}