From 9dcf24eeb7028bc9d13a71ecaebb1eddef3b2a4f Mon Sep 17 00:00:00 2001 From: jigoong Date: Fri, 8 May 2026 22:18:32 +0700 Subject: [PATCH] update config for limit resouce size --- .env.global | 13 + .gitignore | 2 + 03-apiservice-v0.1/.gitignore | 10 - 03-apiservice-v0.1/Dockerfile | 17 - 03-apiservice-v0.1/README.md | 44 -- 03-apiservice-v0.1/app/__init__.py | 0 03-apiservice-v0.1/app/admin.py | 283 ------------- 03-apiservice-v0.1/app/api/__init__.py | 0 03-apiservice-v0.1/app/api/v1/__init__.py | 0 03-apiservice-v0.1/app/api/v1/routes.py | 127 ------ 03-apiservice-v0.1/app/api/v1/schemas.py | 15 - 03-apiservice-v0.1/app/core/__init__.py | 0 03-apiservice-v0.1/app/core/config.py | 35 -- 03-apiservice-v0.1/app/db/__init__.py | 0 03-apiservice-v0.1/app/db/base.py | 5 - 03-apiservice-v0.1/app/db/engine.py | 35 -- 03-apiservice-v0.1/app/db/init_db.py | 13 - 03-apiservice-v0.1/app/db/models.py | 74 ---- 03-apiservice-v0.1/app/main.py | 99 ----- 03-apiservice-v0.1/app/security/__init__.py | 0 03-apiservice-v0.1/app/security/api_key.py | 22 - .../app/security/dependencies.py | 66 --- 03-apiservice-v0.1/docker-compose.yml | 37 -- 03-apiservice-v0.1/requirements.txt | 16 - 05-airflow/config/airflow.cfg | 14 +- 05-airflow/docker-compose.yaml | 42 +- 06-analytics/superset_config.py | 53 ++- REMOTE_HOSTS_DOZZLE_SETUP.md | 400 ++++++++++++++++++ 28 files changed, 497 insertions(+), 925 deletions(-) delete mode 100644 03-apiservice-v0.1/.gitignore delete mode 100644 03-apiservice-v0.1/Dockerfile delete mode 100644 03-apiservice-v0.1/README.md delete mode 100644 03-apiservice-v0.1/app/__init__.py delete mode 100644 03-apiservice-v0.1/app/admin.py delete mode 100644 03-apiservice-v0.1/app/api/__init__.py delete mode 100644 03-apiservice-v0.1/app/api/v1/__init__.py delete mode 100644 03-apiservice-v0.1/app/api/v1/routes.py delete mode 100644 03-apiservice-v0.1/app/api/v1/schemas.py delete mode 100644 03-apiservice-v0.1/app/core/__init__.py delete mode 100644 03-apiservice-v0.1/app/core/config.py delete mode 100644 03-apiservice-v0.1/app/db/__init__.py delete mode 100644 03-apiservice-v0.1/app/db/base.py delete mode 100644 03-apiservice-v0.1/app/db/engine.py delete mode 100644 03-apiservice-v0.1/app/db/init_db.py delete mode 100644 03-apiservice-v0.1/app/db/models.py delete mode 100644 03-apiservice-v0.1/app/main.py delete mode 100644 03-apiservice-v0.1/app/security/__init__.py delete mode 100644 03-apiservice-v0.1/app/security/api_key.py delete mode 100644 03-apiservice-v0.1/app/security/dependencies.py delete mode 100644 03-apiservice-v0.1/docker-compose.yml delete mode 100644 03-apiservice-v0.1/requirements.txt create mode 100644 REMOTE_HOSTS_DOZZLE_SETUP.md diff --git a/.env.global b/.env.global index a8d2678..b9103ba 100644 --- a/.env.global +++ b/.env.global @@ -4,6 +4,7 @@ TZ=Asia/Bangkok DB_HOST=postgres DB_PORT=5432 +DB_PORT_EXPOSE=5435 DB_USER=postgres DB_PASSWORD=Secure_Hospital_Pass_2026 DB_NAME=postgres @@ -13,6 +14,7 @@ POSTGRES_PASSWORD=Secure_Hospital_Pass_2026 KEYCLOAK_ADMIN=admin KEYCLOAK_ADMIN_PASSWORD=admin_secret_pass_2026 +KEYCLOAK_DB_NAME=keycloak SUPERSET_SECRET_KEY=superset_random_secret_key_change_me_2026 SUPERSET_ADMIN_USERNAME=admin @@ -29,3 +31,14 @@ AIRBYTE_PORT=8030 AIRBYTE_BASIC_AUTH_USERNAME= AIRBYTE_BASIC_AUTH_PASSWORD= AIRBYTE_BASIC_AUTH_PROXY_TIMEOUT=900 + +# Dozzle - Docker Log Viewer & Monitoring +DOZZLE_PORT=9999 +DOZZLE_LEVEL=info +DOZZLE_BASE=/dozzle +DOZZLE_HOSTNAME=Sriphat Main Server +DOZZLE_AUTH_PROVIDER=none +DOZZLE_RESTART_POLICY=unless-stopped +# Remote agents: Airbyte and Airflow on 192.168.100.9 +# Format: host:port,host:port (comma-separated) +DOZZLE_REMOTE_AGENT=192.168.100.9:7007 diff --git a/.gitignore b/.gitignore index f5342b5..4444bc9 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ ruff_cache/ */data/ 01-infra/letsencrypt/ .windsurf/ +_daily-log/ +daily-log/ diff --git a/03-apiservice-v0.1/.gitignore b/03-apiservice-v0.1/.gitignore deleted file mode 100644 index 8d1428b..0000000 --- a/03-apiservice-v0.1/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -.env -__pycache__/ -*.pyc -.venv/ -venv/ -.python-version -.pytest_cache/ -.mypy_cache/ -ruff_cache/ -.windsurf/ diff --git a/03-apiservice-v0.1/Dockerfile b/03-apiservice-v0.1/Dockerfile deleted file mode 100644 index e75b84d..0000000 --- a/03-apiservice-v0.1/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.12-slim - -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 - -WORKDIR /app - -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir -r /app/requirements.txt - -COPY ./app /app/app - -ENV TZ=Asia/Bangkok - -EXPOSE 8040 - -CMD ["gunicorn","-k","uvicorn.workers.UvicornWorker","app.main:app","--bind","0.0.0.0:8040","--workers","2","--access-logfile","-","--error-logfile","-"] diff --git a/03-apiservice-v0.1/README.md b/03-apiservice-v0.1/README.md deleted file mode 100644 index 8e303ff..0000000 --- a/03-apiservice-v0.1/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# 03-apiservice: Custom FastAPI Service - -## Build & Start -```bash -docker compose --env-file ../.env.global up --build -d -``` - -## Access -Internal only - access via Nginx Proxy Manager at `/apiservice` - -## Admin UI -- Login: http:///apiservice/admin/ -- Generate API Key: POST /apiservice/admin/api-keys/generate - -## env -env that important for provision -``` -## supabase -SUPABASE_DB_HOST=sdp-db -SUPABASE_DB_PORT=5432 -SUPABASE_DB_USER=postgres.1 -SUPABASE_DB_PASSWORD= -SUPABASE_DB_NAME=postgres -SUPABASE_DB_SSLMODE=disable - -## pgsql -DB_HOST=postgres -DB_PORT=5432 -DB_USER=postgres -DB_PASSWORD= -DB_NAME=postgres -DB_SSLMODE=disable -AIRBYTE_DB_NAME=airbyte -KEYCLOAK_DB_NAME=keycloack -SUPERSET_DB_NAME=superset -#TEMPORAL_DB_NAME=temporal - -## api -ROOT_PATH=/apiservice -APP_NAME=APIsService -ADMIN_SECRET_KEY= -ADMIN_USERNAME=admin -ADMIN_PASSWORD= -``` diff --git a/03-apiservice-v0.1/app/__init__.py b/03-apiservice-v0.1/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/03-apiservice-v0.1/app/admin.py b/03-apiservice-v0.1/app/admin.py deleted file mode 100644 index 1e89fdc..0000000 --- a/03-apiservice-v0.1/app/admin.py +++ /dev/null @@ -1,283 +0,0 @@ -from __future__ import annotations - -from fastapi import HTTPException, Request, status -from fastapi.staticfiles import StaticFiles -from sqladmin import Admin, ModelView -from sqladmin.authentication import AuthenticationBackend -from starlette.responses import HTMLResponse, RedirectResponse -from starlette.datastructures import URL -from sqlalchemy.orm import sessionmaker -from wtforms import BooleanField, SelectField, StringField -from wtforms.validators import Optional - -from app.core.config import settings -from app.db.engine import engine -from app.db.models import ApiClient, ApiKey -from app.security.api_key import generate_api_key, get_prefix, hash_api_key - - -class AdminAuth(AuthenticationBackend): - async def login(self, request: Request) -> bool: - form = await request.form() - username = form.get("username") - password = form.get("password") - - if username == settings.ADMIN_USERNAME and password == settings.ADMIN_PASSWORD: - request.session.update({"admin": True}) - return True - return False - - async def logout(self, request: Request) -> bool: - request.session.clear() - return True - - async def authenticate(self, request: Request) -> bool: - return bool(request.session.get("admin")) - - -class ApiClientAdmin(ModelView, model=ApiClient): - column_list = [ApiClient.id, ApiClient.name, ApiClient.is_active] - - async def insert_model(self, request: Request, data: dict) -> ApiClient: - obj: ApiClient = await super().insert_model(request, data) - - plain_key = generate_api_key() - - db = sessionmaker(bind=engine, autoflush=False, autocommit=False)() - try: - api_key = ApiKey( - client_id=obj.id, - name="auto", - key_prefix=get_prefix(plain_key), - key_hash=hash_api_key(plain_key), - permissions=[], - is_active=True, - ) - db.add(api_key) - db.commit() - db.refresh(api_key) - - request.session["generated_api_key"] = { - "client_id": obj.id, - "client_name": obj.name, - "key_id": api_key.id, - "api_key": plain_key, - } - finally: - db.close() - - return obj - - -class ApiKeyAdmin(ModelView, model=ApiKey): - column_list = [ApiKey.id, ApiKey.client_id, ApiKey.name, ApiKey.is_active, ApiKey.permissions] - form_excluded_columns = [ApiKey.key_hash, ApiKey.key_prefix, ApiKey.created_at] - - form_extra_fields = { - "plain_key": StringField("Plain Key", validators=[Optional()]), - "permissions_csv": StringField("Permissions (comma)", validators=[Optional()]), - "endpoint_path": SelectField("Endpoint", choices=[], validators=[Optional()]), - "perm_read": BooleanField("Read (GET)"), - "perm_write": BooleanField("Write (POST/PATCH)"), - "perm_delete": BooleanField("Delete (DELETE)"), - } - - async def on_model_change(self, data: dict, model: ApiKey, is_created: bool, request: Request) -> None: - plain_key = data.get("plain_key") - if not plain_key and is_created: - plain_key = generate_api_key() - - if plain_key: - model.key_prefix = get_prefix(plain_key) - model.key_hash = hash_api_key(plain_key) - - if is_created: - request.state.generated_api_key_plain = plain_key - - permissions: list[str] = [] - endpoint_path = data.get("endpoint_path") - if endpoint_path: - if data.get("perm_read"): - permissions.append(f"{endpoint_path}:read") - if data.get("perm_write"): - permissions.append(f"{endpoint_path}:write") - if data.get("perm_delete"): - permissions.append(f"{endpoint_path}:delete") - - permissions_csv = data.get("permissions_csv") - if permissions_csv is not None: - perms = [p.strip() for p in permissions_csv.split(",") if p.strip()] - permissions.extend(perms) - - if permissions: - seen: set[str] = set() - deduped: list[str] = [] - for p in permissions: - if p not in seen: - seen.add(p) - deduped.append(p) - model.permissions = deduped - - async def after_model_change(self, data: dict, model: ApiKey, is_created: bool, request: Request) -> None: - if not is_created: - return - - plain_key = getattr(request.state, "generated_api_key_plain", None) - if not plain_key: - return - - request.session["generated_api_key"] = { - "client_id": model.client_id, - "client_name": str(getattr(model, "client", "")) if getattr(model, "client", None) else "", - "key_id": model.id, - "api_key": plain_key, - } - - -def mount_admin(app): - auth_backend = AdminAuth(secret_key=settings.ADMIN_SECRET_KEY) - - class CustomAdmin(Admin): - def get_save_redirect_url( - self, request: Request, form, model_view: ModelView, obj - ): - if ( - getattr(model_view, "model", None) in (ApiClient, ApiKey) - and request.session.get("generated_api_key") - ): - root_path = request.scope.get("root_path") or "" - return URL(f"{root_path}/admin/generated-api-key") - - return super().get_save_redirect_url( - request=request, - form=form, - model_view=model_view, - obj=obj, - ) - - admin = CustomAdmin( - app=app, - engine=engine, - authentication_backend=auth_backend, - title="My Service Management", - base_url="/admin", - ) - - openapi = app.openapi() - paths = openapi.get("paths") or {} - endpoint_choices: list[tuple[str, str]] = [] - for path in sorted(paths.keys()): - if not path.startswith("/api/"): - continue - methods = paths.get(path) or {} - available = sorted([m.upper() for m in methods.keys()]) - label = f"{path} [{' '.join(available)}]" if available else path - endpoint_choices.append((path, label)) - ApiKeyAdmin.form_extra_fields["endpoint_path"].kwargs["choices"] = endpoint_choices - - SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False) - - admin.add_view(ApiClientAdmin) - admin.add_view(ApiKeyAdmin) - - @app.get("/admin/generated-api-key") - async def _admin_generated_api_key(request: Request): - if not request.session.get("admin"): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") - - key_info = request.session.pop("generated_api_key", None) - root_path = request.scope.get("root_path") or "" - clients_url = f"{root_path}/admin/{ApiClientAdmin.identity}/list" - - if not key_info: - return HTMLResponse( - f"

No API key to display

The API key was already shown or expired.

Back to clients

", - status_code=200, - ) - - client_name = key_info.get("client_name", "") - client_id = key_info.get("client_id", "") - key_id = key_info.get("key_id", "") - api_key = key_info.get("api_key", "") - - return HTMLResponse( - ( - "

API key generated

" - "

Copy this API key now. You won't be able to view it again.

" - f"

Client: {client_name} (ID: {client_id})

" - f"

Key ID: {key_id}

" - f"
{api_key}
" - f"

Back to clients

" - ), - status_code=200, - ) - - @app.get("/admin/clients/{client_id}/generate-api-key") - async def _admin_generate_api_key_get( - request: Request, - client_id: int, - permissions: str = "", - name: str | None = None, - ): - if not request.session.get("admin"): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") - - perms = [p.strip() for p in permissions.split(",") if p.strip()] - plain_key = generate_api_key() - - db = SessionLocal() - try: - client = db.get(ApiClient, client_id) - if not client: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Client not found") - - api_key = ApiKey( - client_id=client_id, - name=name, - key_prefix=get_prefix(plain_key), - key_hash=hash_api_key(plain_key), - permissions=perms, - is_active=True, - ) - db.add(api_key) - db.commit() - db.refresh(api_key) - - return {"key_id": api_key.id, "api_key": plain_key, "permissions": perms} - finally: - db.close() - - @app.post("/admin/api-keys/generate") - async def _admin_generate_api_key( - request: Request, - client_id: int, - permissions: str = "", - name: str | None = None, - ): - if not request.session.get("admin"): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") - - perms = [p.strip() for p in permissions.split(",") if p.strip()] - plain_key = generate_api_key() - - db = SessionLocal() - try: - client = db.get(ApiClient, client_id) - if not client: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Client not found") - - api_key = ApiKey( - client_id=client_id, - name=name, - key_prefix=get_prefix(plain_key), - key_hash=hash_api_key(plain_key), - permissions=perms, - is_active=True, - ) - db.add(api_key) - db.commit() - db.refresh(api_key) - - return {"key_id": api_key.id, "api_key": plain_key, "permissions": perms} - finally: - db.close() diff --git a/03-apiservice-v0.1/app/api/__init__.py b/03-apiservice-v0.1/app/api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/03-apiservice-v0.1/app/api/v1/__init__.py b/03-apiservice-v0.1/app/api/v1/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/03-apiservice-v0.1/app/api/v1/routes.py b/03-apiservice-v0.1/app/api/v1/routes.py deleted file mode 100644 index 2274075..0000000 --- a/03-apiservice-v0.1/app/api/v1/routes.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Annotated -from zoneinfo import ZoneInfo - -from fastapi import APIRouter, Depends -from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.orm import Session - -from app.api.v1.schemas import FeedCheckpointIn -from app.core.config import settings -from app.db.models import RawOpdCheckpoint -from app.security.dependencies import get_db, get_supabase_db, require_permission -from app.utils.supabase_client import SupabaseAPIError, upsert_to_supabase_sync - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/api/v1") - -PERM_FEED_CHECKPOINT_WRITE = "/api/v1/feed/checkpoint:write" -PERM_FEED_CHECKPOINT_WRITE_LEGACY = "feed.checkpoint:write" - - -def _to_tz(dt): - if dt is None: - return None - if dt.tzinfo is None: - return dt.replace(tzinfo=ZoneInfo(settings.TIMEZONE)) - return dt.astimezone(ZoneInfo(settings.TIMEZONE)) - - -def _to_iso(dt): - """Convert datetime to ISO 8601 string for Supabase API.""" - if dt is None: - return None - return dt.isoformat() - - -@router.post("/feed/checkpoint") -def upsert_feed_checkpoint( - payload: list[FeedCheckpointIn], - db: Annotated[Session, Depends(get_db)], -): - rows = [] - supabase_rows = [] - - #clean_data = payload.model_dump(exclude_none=True) - for item in payload: - # Prepare data for local database 'default' if item.id is None else - row = { - "id": item.id, - "hn": item.hn, - "vn": item.vn, - "location": item.location, - "type": item.type, - "timestamp_in": _to_tz(item.timestamp_in), - "timestamp_out": _to_tz(item.timestamp_out), - "waiting_time": item.waiting_time, - "bu": item.bu, - } - if item.id is None: - del(row["id"]) - rows.append(row) - - # Prepare data for Supabase API (convert datetime to ISO string) 'default' if item.id is None else - supabase_row = { - "id": item.id, - "hn": item.hn, - "vn": item.vn, - "location": item.location, - "type": item.type, - "timestamp_in": _to_iso(_to_tz(item.timestamp_in)), - "timestamp_out": _to_iso(_to_tz(item.timestamp_out)), - "waiting_time": item.waiting_time, - "bu": item.bu, - } - if item.id is None: - del(supabase_row["id"]) - supabase_rows.append(supabase_row) - - # Insert/update to local database - stmt = insert(RawOpdCheckpoint).values(rows) - update_cols = { - "id": stmt.excluded.id, - "type": stmt.excluded.type, - "timestamp_in": stmt.excluded.timestamp_in, - "timestamp_out": stmt.excluded.timestamp_out, - "waiting_time": stmt.excluded.waiting_time, - "bu": stmt.excluded.bu, - } - - stmt = stmt.on_conflict_do_update( - index_elements=[RawOpdCheckpoint.hn, RawOpdCheckpoint.vn, RawOpdCheckpoint.location, RawOpdCheckpoint.timestamp_in], - set_=update_cols, - ) - result = db.execute(stmt) - db.commit() - - # Send data to Supabase via API call - supabase_result = None - supabase_error = None - - try: - logger.info(f"Sending {len(supabase_rows)} records to Supabase API") - supabase_result = upsert_to_supabase_sync( - table="raw_opd_checkpoint", - data=supabase_rows, - on_conflict="hn,vn,location,timestamp_in", - ) - logger.info(f"Successfully sent data to Supabase: {supabase_result.get('status_code')}") - except SupabaseAPIError as e: - logger.error(f"Failed to send data to Supabase: {str(e)}") - supabase_error = str(e) - except Exception as e: - logger.error(f"Unexpected error sending data to Supabase: {str(e)}") - supabase_error = f"Unexpected error: {str(e)}" - - return { - "upserted": len(rows), - "rowcount": result.rowcount, - "supabase": { - "success": supabase_result is not None, - "result": supabase_result, - "error": supabase_error, - }, - } diff --git a/03-apiservice-v0.1/app/api/v1/schemas.py b/03-apiservice-v0.1/app/api/v1/schemas.py deleted file mode 100644 index 524ae1f..0000000 --- a/03-apiservice-v0.1/app/api/v1/schemas.py +++ /dev/null @@ -1,15 +0,0 @@ -from datetime import datetime - -from pydantic import BaseModel - - -class FeedCheckpointIn(BaseModel): - id: int | None = None - hn: int - vn: int - location: str - type: str - timestamp_in: datetime - timestamp_out: datetime | None = None - waiting_time: int | None = None - bu: str | None = None diff --git a/03-apiservice-v0.1/app/core/__init__.py b/03-apiservice-v0.1/app/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/03-apiservice-v0.1/app/core/config.py b/03-apiservice-v0.1/app/core/config.py deleted file mode 100644 index 4f50d3f..0000000 --- a/03-apiservice-v0.1/app/core/config.py +++ /dev/null @@ -1,35 +0,0 @@ -from pydantic_settings import BaseSettings, SettingsConfigDict - - -class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env", extra="ignore") - - APP_NAME: str = "APIsService" - - DB_HOST: str - DB_PORT: int = 5432 - DB_USER: str - DB_PASSWORD: str - DB_NAME: str - DB_SSLMODE: str = "disable" - - SUPABASE_DB_HOST: str - SUPABASE_DB_PORT: int = 5432 - SUPABASE_DB_USER: str - SUPABASE_DB_PASSWORD: str - SUPABASE_DB_NAME: str - SUPABASE_DB_SSLMODE: str = "disable" - - SUPABASE_API_URL: str - SUPABASE_API_KEY: str - - ROOT_PATH: str = "" - - TIMEZONE: str = "Asia/Bangkok" - - ADMIN_SECRET_KEY: str - ADMIN_USERNAME: str - ADMIN_PASSWORD: str - - -settings = Settings() diff --git a/03-apiservice-v0.1/app/db/__init__.py b/03-apiservice-v0.1/app/db/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/03-apiservice-v0.1/app/db/base.py b/03-apiservice-v0.1/app/db/base.py deleted file mode 100644 index fa2b68a..0000000 --- a/03-apiservice-v0.1/app/db/base.py +++ /dev/null @@ -1,5 +0,0 @@ -from sqlalchemy.orm import DeclarativeBase - - -class Base(DeclarativeBase): - pass diff --git a/03-apiservice-v0.1/app/db/engine.py b/03-apiservice-v0.1/app/db/engine.py deleted file mode 100644 index a5fde95..0000000 --- a/03-apiservice-v0.1/app/db/engine.py +++ /dev/null @@ -1,35 +0,0 @@ -from urllib.parse import quote_plus - -from sqlalchemy import create_engine - -from app.core.config import settings - - -def build_db_url() -> str: - user = quote_plus(settings.DB_USER) - password = quote_plus(settings.DB_PASSWORD) - host = settings.DB_HOST - port = settings.DB_PORT - db = quote_plus(settings.DB_NAME) - - return ( - f"postgresql+psycopg://{user}:{password}@{host}:{port}/{db}" - f"?sslmode={quote_plus(settings.DB_SSLMODE)}" - ) - - -def build_supabase_db_url() -> str: - user = quote_plus(settings.SUPABASE_DB_USER) - password = quote_plus(settings.SUPABASE_DB_PASSWORD) - host = settings.SUPABASE_DB_HOST - port = settings.SUPABASE_DB_PORT - db = quote_plus(settings.SUPABASE_DB_NAME) - - return ( - f"postgresql+psycopg://{user}:{password}@{host}:{port}/{db}" - f"?sslmode={quote_plus(settings.SUPABASE_DB_SSLMODE)}" - ) - - -engine = create_engine(build_db_url(), pool_pre_ping=True) -supabase_engine = create_engine(build_supabase_db_url(), pool_pre_ping=True) diff --git a/03-apiservice-v0.1/app/db/init_db.py b/03-apiservice-v0.1/app/db/init_db.py deleted file mode 100644 index 7f83020..0000000 --- a/03-apiservice-v0.1/app/db/init_db.py +++ /dev/null @@ -1,13 +0,0 @@ -from sqlalchemy import text - -from app.db.base import Base -from app.db.engine import engine - - -def init_db() -> None: - # with engine.begin() as conn: - # conn.execute(text("CREATE SCHEMA IF NOT EXISTS fastapi")) - # conn.execute(text("CREATE SCHEMA IF NOT EXISTS operationbi")) - - # Base.metadata.create_all(bind=conn) - pass \ No newline at end of file diff --git a/03-apiservice-v0.1/app/db/models.py b/03-apiservice-v0.1/app/db/models.py deleted file mode 100644 index f5a43f9..0000000 --- a/03-apiservice-v0.1/app/db/models.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from datetime import datetime - -from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.orm import Mapped, mapped_column, relationship - -from app.db.base import Base - - -class RawOpdCheckpoint(Base): - __tablename__ = "raw_opd_checkpoint" - __table_args__ = ( - UniqueConstraint("hn", "vn", "location", name="uq_raw_opd_checkpoint_hn_vn_location"), - {"schema": "rawdata"}, - ) - - id: Mapped[int] = mapped_column(BigInteger, primary_key=True) - hn: Mapped[int] = mapped_column(BigInteger, nullable=False) - vn: Mapped[int] = mapped_column(BigInteger, nullable=False) - location: Mapped[str] = mapped_column(Text, nullable=False) - type: Mapped[str] = mapped_column(String(64), nullable=False) - timestamp_in: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - timestamp_out: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - waiting_time: Mapped[int | None] = mapped_column(Integer, nullable=True) - bu: Mapped[str | None] = mapped_column(String(128), nullable=True) - - -class ApiClient(Base): - __tablename__ = "api_client" - __table_args__ = {"schema": "fastapi"} - - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - name: Mapped[str] = mapped_column(String(128), unique=True, nullable=False) - is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) - - api_keys: Mapped[list[ApiKey]] = relationship( - back_populates="client", - cascade="all, delete-orphan", - passive_deletes=True, - ) - - def __str__(self) -> str: - client_id = getattr(self, "id", None) - if client_id is None: - return self.name - return f"{self.name} ({client_id})" - - def __repr__(self) -> str: - return str(self) - - -class ApiKey(Base): - __tablename__ = "api_key" - __table_args__ = {"schema": "fastapi"} - - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - client_id: Mapped[int] = mapped_column( - ForeignKey("fastapi.api_client.id", ondelete="CASCADE"), nullable=False - ) - name: Mapped[str | None] = mapped_column(String(128), nullable=True) - - key_prefix: Mapped[str] = mapped_column(String(12), nullable=False) - key_hash: Mapped[str] = mapped_column(Text, nullable=False) - - permissions: Mapped[list[str]] = mapped_column(JSONB, nullable=False, default=list) - is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) - - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - client: Mapped[ApiClient] = relationship(back_populates="api_keys") diff --git a/03-apiservice-v0.1/app/main.py b/03-apiservice-v0.1/app/main.py deleted file mode 100644 index 88b7d3b..0000000 --- a/03-apiservice-v0.1/app/main.py +++ /dev/null @@ -1,99 +0,0 @@ -from contextlib import asynccontextmanager - -from fastapi import FastAPI -from starlette.datastructures import Headers -from starlette.middleware.sessions import SessionMiddleware -from fastapi.middleware.cors import CORSMiddleware -from starlette.middleware.base import BaseHTTPMiddleware - -class ForceHTTPSMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request, call_next): - # บังคับให้ FastAPI มองว่า Request ที่เข้ามาเป็น HTTPS เสมอ - # เพื่อให้ url_for() เจนลิงก์ CSS/JS เป็น https:// - request.scope["scheme"] = "https" - response = await call_next(request) - return response - - -class ForwardedProtoMiddleware: - def __init__(self, app): - self.app = app - - async def __call__(self, scope, receive, send): - if scope["type"] in {"http", "websocket"}: - headers = Headers(scope=scope) - forwarded_proto = headers.get("x-forwarded-proto") - if forwarded_proto: - proto = forwarded_proto.split(",", 1)[0].strip() - if proto: - new_scope = dict(scope) - new_scope["scheme"] = proto - return await self.app(new_scope, receive, send) - - return await self.app(scope, receive, send) - - -# class RootPathStripMiddleware: -# def __init__(self, app, prefix: str): -# self.app = app -# self.prefix = (prefix or "").rstrip("/") - -# async def __call__(self, scope, receive, send): -# if scope["type"] in {"http", "websocket"} and self.prefix: -# path = scope.get("path") or "" -# new_scope = dict(scope) -# new_scope["root_path"] = self.prefix - -# if path == self.prefix or path.startswith(self.prefix + "/"): -# new_path = path[len(self.prefix) :] -# new_scope["path"] = new_path if new_path else "/" - -# return await self.app(new_scope, receive, send) - -# return await self.app(scope, receive, send) - -from app.admin import mount_admin -from app.api.v1.routes import router as v1_router -from app.core.config import settings -from app.db.init_db import init_db -from fastapi.staticfiles import StaticFiles -from sqladmin import Admin -import os -import sqladmin - -# รายชื่อ Origins ที่อนุญาตให้ยิง API มาหาเราได้ -origins = [ - "http://localhost:80400", # สำหรับตอนพัฒนา Frontend - "https://ai.sriphat.com", # Domain หลักของคุณ - "http://ai.sriphat.com", -] - -@asynccontextmanager -async def lifespan(_: FastAPI): - init_db() - yield - -print(settings.ROOT_PATH, flush=True) - -sqladmin_dir = os.path.dirname(sqladmin.__file__) -statics_path = os.path.join(sqladmin_dir, "statics") - -app = FastAPI(title=settings.APP_NAME, root_path=settings.ROOT_PATH, lifespan=lifespan) -#if settings.ROOT_PATH: -# app.add_middleware(RootPathStripMiddleware, prefix=settings.ROOT_PATH) -app.add_middleware(ForceHTTPSMiddleware) -app.add_middleware(SessionMiddleware, secret_key=settings.ADMIN_SECRET_KEY) -app.add_middleware(ForwardedProtoMiddleware) -app.include_router(v1_router) -app.mount("/admin/statics", StaticFiles(directory=statics_path), name="admin_statics") -app.mount("/apiservice/admin/statics", StaticFiles(directory=statics_path), name="proxy_admin_statics") -app.add_middleware( - CORSMiddleware, - allow_origins=origins, # หรือ ["*"] ถ้าต้องการอนุญาตทั้งหมด (ไม่แนะนำใน production) - allow_credentials=True, # สำคัญมาก! ต้องเป็น True ถ้าหน้า Admin/API มีการใช้ Cookies/Sessions - allow_methods=["*"], # อนุญาตทุก HTTP Method (GET, POST, PUT, DELETE, etc.) - allow_headers=["*"], # อนุญาตทุก Headers -) - -mount_admin(app) - diff --git a/03-apiservice-v0.1/app/security/__init__.py b/03-apiservice-v0.1/app/security/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/03-apiservice-v0.1/app/security/api_key.py b/03-apiservice-v0.1/app/security/api_key.py deleted file mode 100644 index 9a28d98..0000000 --- a/03-apiservice-v0.1/app/security/api_key.py +++ /dev/null @@ -1,22 +0,0 @@ -import secrets - -import bcrypt - - -def generate_api_key(prefix_len: int = 8, token_bytes: int = 32) -> str: - prefix = secrets.token_urlsafe(prefix_len)[:prefix_len] - token = secrets.token_urlsafe(token_bytes) - return f"{prefix}.{token}" - - -def get_prefix(api_key: str) -> str: - return api_key.split(".", 1)[0] - - -def hash_api_key(api_key: str) -> str: - hashed = bcrypt.hashpw(api_key.encode("utf-8"), bcrypt.gensalt()) - return hashed.decode("utf-8") - - -def verify_api_key(api_key: str, api_key_hash: str) -> bool: - return bcrypt.checkpw(api_key.encode("utf-8"), api_key_hash.encode("utf-8")) diff --git a/03-apiservice-v0.1/app/security/dependencies.py b/03-apiservice-v0.1/app/security/dependencies.py deleted file mode 100644 index 1efe2fd..0000000 --- a/03-apiservice-v0.1/app/security/dependencies.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import Annotated -from collections.abc import Sequence - -from fastapi import Depends, HTTPException, Request, status -from sqlalchemy import select -from sqlalchemy.orm import Session, sessionmaker - -from app.db.engine import engine, supabase_engine -from app.db.models import ApiKey -from app.security.api_key import get_prefix, verify_api_key - - -SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False) -SupabaseSessionLocal = sessionmaker(bind=supabase_engine, autoflush=False, autocommit=False) - - -def get_db(): - db = SessionLocal() - try: - yield db - finally: - db.close() - - -def get_supabase_db(): - db = SupabaseSessionLocal() - try: - yield db - finally: - db.close() - - -def get_bearer_token(request: Request) -> str: - auth = request.headers.get("authorization") - if not auth: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Missing Authorization") - - parts = auth.split(" ", 1) - if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip(): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid Authorization") - - return parts[1].strip() - - -def require_permission(permission: str | Sequence[str]): - def _dep( - token: Annotated[str, Depends(get_bearer_token)], - db: Annotated[Session, Depends(get_db)], - ) -> ApiKey: - prefix = get_prefix(token) - stmt = select(ApiKey).where(ApiKey.key_prefix == prefix, ApiKey.is_active.is_(True)) - api_key = db.execute(stmt).scalar_one_or_none() - if not api_key: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key") - - if not verify_api_key(token, api_key.key_hash): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key") - - allowed = set(api_key.permissions or []) - required = [permission] if isinstance(permission, str) else list(permission) - if not any(p in allowed for p in required): - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Permission denied") - - return api_key - - return _dep diff --git a/03-apiservice-v0.1/docker-compose.yml b/03-apiservice-v0.1/docker-compose.yml deleted file mode 100644 index b52b08c..0000000 --- a/03-apiservice-v0.1/docker-compose.yml +++ /dev/null @@ -1,37 +0,0 @@ -services: - apiservice: - build: . - container_name: apiservice - env_file: - - ../.env.global - environment: - - TZ=${TZ:-Asia/Bangkok} - - DB_HOST=${DB_HOST} - - DB_PORT=${DB_PORT} - - DB_USER=${DB_USER} - - DB_PASSWORD=${DB_PASSWORD} - - DB_NAME=${DB_NAME} - - DB_SSLMODE=${DB_SSLMODE} - - ROOT_PATH=${ROOT_PATH} - - APP_NAME=${APP_NAME} - - ADMIN_SECRET_KEY=${ADMIN_SECRET_KEY} - - ADMIN_USERNAME=${ADMIN_USERNAME} - - ADMIN_PASSWORD=${ADMIN_PASSWORD} - networks: - - shared_data_network - restart: unless-stopped - volumes: - - ./app:/app/app - - .env:/app/.env - ports: - - 0.0.0.0:8040:8040 - healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8040/apiservice/docs', timeout=5).read()"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - -networks: - shared_data_network: - external: true diff --git a/03-apiservice-v0.1/requirements.txt b/03-apiservice-v0.1/requirements.txt deleted file mode 100644 index f04f728..0000000 --- a/03-apiservice-v0.1/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -fastapi==0.115.8 -uvicorn==0.34.0 -gunicorn==23.0.0 -SQLAlchemy==2.0.38 -psycopg==3.2.5 -pydantic==2.10.6 -pydantic-settings==2.7.1 -psycopg[binary] -sqladmin==0.20.1 -itsdangerous==2.2.0 -bcrypt==4.3.0 -python-multipart==0.0.20 -httpx==0.28.1 -WTForms -#==3.2.1 - diff --git a/05-airflow/config/airflow.cfg b/05-airflow/config/airflow.cfg index 92d8161..67a06b8 100644 --- a/05-airflow/config/airflow.cfg +++ b/05-airflow/config/airflow.cfg @@ -40,7 +40,7 @@ might_contain_dag_callable = airflow.utils.file.might_contain_dag_via_default_he # # Variable: AIRFLOW__CORE__DEFAULT_TIMEZONE # -default_timezone = utc +default_timezone = Asia/Bangkok # The executor class that airflow should use. Choices include # ``LocalExecutor``, ``CeleryExecutor``, @@ -90,7 +90,7 @@ simple_auth_manager_all_admins = False # # Variable: AIRFLOW__CORE__PARALLELISM # -parallelism = 8 +parallelism = 2 # The maximum number of task instances allowed to run concurrently in each dag run. # This is also configurable per-dag with ``max_active_tasks``, @@ -115,7 +115,7 @@ dags_are_paused_at_creation = True # # Variable: AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG # -max_active_runs_per_dag = 16 +max_active_runs_per_dag = 1 # (experimental) The maximum number of consecutive DAG failures before DAG is automatically paused. # This is also configurable per DAG level with ``max_consecutive_failed_dag_runs``, @@ -2166,7 +2166,7 @@ refresh_interval = 300 # # Variable: AIRFLOW__DAG_PROCESSOR__PARSING_PROCESSES # -parsing_processes = 2 +parsing_processes = 1 # One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``. # The DAG processor will list and sort the dag files to decide the parsing order. @@ -2193,7 +2193,9 @@ max_callbacks_per_loop = 20 # # Variable: AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL # -min_file_process_interval = 30 +min_file_process_interval = 90 + +dag_dir_list_interval = 90 # How long (in seconds) to wait after we have re-parsed a DAG file before deactivating stale # DAGs (DAGs which are no longer present in the expected files). The reason why we need @@ -2491,7 +2493,7 @@ flower_basic_auth = # # Variable: AIRFLOW__CELERY__SYNC_PARALLELISM # -sync_parallelism = 0 +sync_parallelism = 2 # Import path for celery configuration options # diff --git a/05-airflow/docker-compose.yaml b/05-airflow/docker-compose.yaml index 86b4b4c..93f3572 100644 --- a/05-airflow/docker-compose.yaml +++ b/05-airflow/docker-compose.yaml @@ -64,7 +64,7 @@ x-airflow-common: AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY:-} AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' - AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES:-'false'} + AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES:-False} AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' # yamllint disable rule:line-length # Use simple http server on scheduler for health checks @@ -76,18 +76,22 @@ x-airflow-common: _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} # The following line can be used to set a custom config file, stored in the local config folder AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' + AIRFLOW__WEBSERVER__BASE_URL: ${AIRFLOW__WEBSERVER__BASE_URL:-https://ai.sriphat.com/airflow} + AIRFLOW__API__BASE_URL: ${AIRFLOW__WEBSERVER__BASE_URL:-https://ai.sriphat.com/airflow} + AIRFLOW__WEBSERVER__WEB_SERVER_PORT: ${AIRFLOW__WEBSERVER__WEB_SERVER_PORT:-8080} volumes: - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins user: "${AIRFLOW_UID:-50000}:0" - depends_on: - &airflow-common-depends-on - #airflow-base: + x-depends_on: + &airflow-common-depends-on + {} + # airflow-base: # condition: service_completed_successfully - redis: - condition: service_healthy + # redis: + # condition: service_healthy networks: - shared_data_network @@ -114,19 +118,19 @@ services: # start_period: 5s # restart: always - redis: - # Redis is limited to 7.2-bookworm due to licencing change - # https://redis.io/blog/redis-adopts-dual-source-available-licensing/ - image: redis:7.2-bookworm - expose: - - 6379 - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 30s - retries: 50 - start_period: 30s - restart: always + # redis: + # # Redis is limited to 7.2-bookworm due to licencing change + # # https://redis.io/blog/redis-adopts-dual-source-available-licensing/ + # image: redis:7.2-bookworm + # expose: + # - 6379 + # healthcheck: + # test: ["CMD", "redis-cli", "ping"] + # interval: 10s + # timeout: 30s + # retries: 50 + # start_period: 30s + # restart: always airflow-apiserver: <<: *airflow-common diff --git a/06-analytics/superset_config.py b/06-analytics/superset_config.py index e003e3f..4294778 100644 --- a/06-analytics/superset_config.py +++ b/06-analytics/superset_config.py @@ -2,9 +2,58 @@ import os SECRET_KEY = os.environ.get('SUPERSET_SECRET_KEY') SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg2://{os.environ.get('DATABASE_USER')}:{os.environ.get('DATABASE_PASSWORD')}@{os.environ.get('DATABASE_HOST')}:{os.environ.get('DATABASE_PORT')}/{os.environ.get('DATABASE_DB')}" - ENABLE_PROXY_FIX = True PUBLIC_ROLE_LIKE = "Gamma" -WTF_CSRF_ENABLED = True +WTF_CSRF_ENABLED = False WTF_CSRF_TIME_LIMIT = None + +FEATURE_FLAGS = { + "EMBEDDED_SUPERSET": True, +} + +GUEST_ROLE_NAME = "Gamma" + +ENABLE_CORS = True +CORS_OPTIONS = { + 'supports_credentials': True, + 'allow_headers': ['*'], + 'resources': ['*'], + 'origins': ['*'] +} + +SESSION_COOKIE_SAMESITE = "Lax" +SESSION_COOKIE_SECURE = False + +GUEST_TOKEN_JWT_SECRET = 'RgSCvATmH8fzluoFB6cqkdCXsY7jjq/zwGLRatoxYtI=' +GUEST_TOKEN_JWT_EXP_SECONDS = 86400 # 24 hours + +# Logo link configuration +LOGO_TARGET_PATH = '/superset/welcome/' + +# Embedded SDK Configuration +EMBEDDED_SUPERSET = True +TALISMAN_ENABLED = False +ENABLE_TEMPLATE_PROCESSING = True + +# Guest token configuration for embedded SDK +GUEST_TOKEN_JWT_ALGORITHM = "HS256" +GUEST_TOKEN_JWT_EXP_SECONDS = 300 # 5 minutes + +# Domain whitelist for embedded dashboards +WEBDRIVER_BASEURL_USER_FRIENDLY_NAME = "Sriphat Dashboard" + + +# Embedded SDK Domain Whitelist +EMBEDDED_SDK_HOST_WHITELIST = [ + "http://localhost:8800", + "https://ai.sriphat.com", + "http://127.0.0.1:8800" +] + +# Allow embedding from specific domains +TALISMAN_ALLOWED_DOMAINS = [ + "http://localhost:8800", + "https://ai.sriphat.com", + "http://127.0.0.1:8800" +] diff --git a/REMOTE_HOSTS_DOZZLE_SETUP.md b/REMOTE_HOSTS_DOZZLE_SETUP.md new file mode 100644 index 0000000..d774ac0 --- /dev/null +++ b/REMOTE_HOSTS_DOZZLE_SETUP.md @@ -0,0 +1,400 @@ +# Dozzle Multi-Host Setup Guide + +คู่มือการตั้งค่า Dozzle สำหรับ monitor Docker containers บนหลาย hosts + +## 🏗️ Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Main Server (Current Host) │ +│ ├─ Nginx Proxy Manager │ +│ ├─ Keycloak │ +│ ├─ PostgreSQL │ +│ ├─ API Service │ +│ ├─ Supabase │ +│ ├─ Superset │ +│ └─ Dozzle (Main UI) ──────────────┐ │ +└───────────────────────────────────┼─────────────────────────┘ + │ + ┌───────────────┴───────────────┐ + │ │ + ┌───────────▼──────────┐ ┌────────────▼─────────┐ + │ 192.168.100.9 │ │ 192.168.100.9 │ + │ Airbyte Host │ │ Airflow Host │ + │ ├─ Airbyte Services │ │ ├─ Airflow Services │ + │ └─ Dozzle Agent │ │ └─ Dozzle Agent │ + │ (Port 7007) │ │ (Port 7008) │ + └──────────────────────┘ └──────────────────────┘ +``` + +## 📋 Setup Steps + +### **Step 1: ติดตั้ง Dozzle Agent บน Remote Hosts** + +#### **สำหรับ Airbyte Host (192.168.100.9:7007)** + +สร้าง/แก้ไข `docker-compose.yml` ใน Airbyte directory: + +```yaml +services: + # ... existing Airbyte services ... + + dozzle-agent: + image: amir20/dozzle:latest + container_name: dozzle-agent-airbyte + command: agent + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "7007:7007" + environment: + DOZZLE_LEVEL: info + DOZZLE_HOSTNAME: Airbyte Server + TZ: Asia/Bangkok + restart: unless-stopped + networks: + - airbyte_network # ใช้ network ของ Airbyte +``` + +**Start agent:** +```bash +docker compose up -d dozzle-agent +``` + +#### **สำหรับ Airflow Host (192.168.100.9:7008)** + +สร้าง/แก้ไข `docker-compose.yml` ใน Airflow directory: + +```yaml +services: + # ... existing Airflow services ... + + dozzle-agent: + image: amir20/dozzle:latest + container_name: dozzle-agent-airflow + command: agent + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "7008:7007" # External: 7008, Internal: 7007 + environment: + DOZZLE_LEVEL: info + DOZZLE_HOSTNAME: Airflow Server + TZ: Asia/Bangkok + restart: unless-stopped + networks: + - shared_data_network # ใช้ network ของ Airflow +``` + +**Start agent:** +```bash +docker compose up -d dozzle-agent +``` + +### **Step 2: ตรวจสอบ Agents** + +```bash +# ตรวจสอบ Airbyte agent +curl http://192.168.100.9:7007/healthcheck + +# ตรวจสอบ Airflow agent +curl http://192.168.100.9:7008/healthcheck + +# ดู logs +docker logs dozzle-agent-airbyte +docker logs dozzle-agent-airflow +``` + +### **Step 3: Start Dozzle Main UI (Main Server)** + +```bash +cd 01-infra +docker compose up -d dozzle + +# ตรวจสอบ +docker logs dozzle -f +``` + +### **Step 4: เข้าใช้งาน** + +**Direct access:** +``` +http://localhost:9999/dozzle +``` + +**ผ่าน Nginx:** +``` +http://ai.sriphat.com/dozzle +``` + +## 🔧 Configuration Details + +### **Main Server (.env.global)** + +```bash +# Dozzle - Docker Log Viewer & Monitoring +DOZZLE_PORT=9999 +DOZZLE_LEVEL=info +DOZZLE_BASE=/dozzle +DOZZLE_HOSTNAME=Sriphat Main Server +DOZZLE_AUTH_PROVIDER=none +DOZZLE_RESTART_POLICY=unless-stopped + +# Remote agents: Airbyte and Airflow on 192.168.100.9 +# Format: host:port,host:port (comma-separated) +DOZZLE_REMOTE_AGENT=192.168.100.9:7007,192.168.100.9:7008 +``` + +### **Agent Configuration** + +**Airbyte Agent:** +- Port: 7007 +- Hostname: Airbyte Server +- Monitors: Airbyte containers + +**Airflow Agent:** +- Port: 7008 +- Hostname: Airflow Server +- Monitors: Airflow containers + +## 🌐 Nginx Configuration + +Dozzle config ถูกเพิ่มใน: +- `01-infra/nginx-configs/dozzle.conf` +- `01-infra/nginx-configs/complete-example.conf` + +**ตั้งค่าใน Nginx Proxy Manager:** +1. ไปที่ Proxy Host → Edit +2. Tab "Advanced" +3. เพิ่ม Dozzle config จาก `complete-example.conf` + +## 🔍 Features + +### **1. Multi-Host Monitoring** +- ✅ ดู logs จาก Main Server +- ✅ ดู logs จาก Airbyte Host (192.168.100.9:7007) +- ✅ ดู logs จาก Airflow Host (192.168.100.9:7008) +- ✅ Switch ระหว่าง hosts ผ่าน dropdown + +### **2. Real-time Log Streaming** +- Live log updates +- Color-coded logs +- JSON formatting +- Multi-line grouping + +### **3. Container Management** +- View container stats (CPU, Memory, Network) +- Start/Stop/Restart containers +- Interactive shell access +- Container filtering + +### **4. Advanced Features** +- Search และ filter logs +- Download logs +- Multiple container view +- SQL-based log querying + +## 🐛 Troubleshooting + +### **Issue: Agent ไม่ปรากฏใน UI** + +**ตรวจสอบ:** +```bash +# 1. Agent ทำงานหรือไม่ +docker ps | grep dozzle-agent + +# 2. Port เปิดหรือไม่ +netstat -tulpn | grep 7007 +netstat -tulpn | grep 7008 + +# 3. Firewall +sudo ufw status +sudo ufw allow 7007 +sudo ufw allow 7008 + +# 4. Network connectivity +ping 192.168.100.9 +telnet 192.168.100.9 7007 +telnet 192.168.100.9 7008 +``` + +### **Issue: Connection Refused** + +**สาเหตุ:** +- Agent ไม่ทำงาน +- Firewall block port +- Network ไม่เชื่อมต่อ + +**วิธีแก้:** +```bash +# Restart agent +docker restart dozzle-agent-airbyte +docker restart dozzle-agent-airflow + +# ตรวจสอบ logs +docker logs dozzle-agent-airbyte +docker logs dozzle-agent-airflow + +# ทดสอบ connectivity +curl http://192.168.100.9:7007/healthcheck +curl http://192.168.100.9:7008/healthcheck +``` + +### **Issue: Containers ไม่แสดงใน Agent** + +**สาเหตุ:** +- Docker socket ไม่ mount +- Agent ไม่มี permission + +**วิธีแก้:** +```bash +# ตรวจสอบ volume mount +docker inspect dozzle-agent-airbyte | grep docker.sock + +# ตรวจสอบ permissions +ls -la /var/run/docker.sock + +# Restart agent +docker restart dozzle-agent-airbyte +``` + +## 🔐 Security Considerations + +### **1. Network Security** + +**ใช้ Internal Network (แนะนำ):** +```yaml +# Agent ไม่ expose port ออกภายนอก +# ใช้ Docker network แทน +dozzle-agent: + # ไม่ต้องมี ports section + networks: + - shared_data_network +``` + +**Main UI เชื่อมต่อผ่าน network:** +```yaml +DOZZLE_REMOTE_AGENT=dozzle-agent-airbyte:7007,dozzle-agent-airflow:7007 +``` + +### **2. Firewall Rules** + +```bash +# อนุญาตเฉพาะ Main Server +sudo ufw allow from to any port 7007 +sudo ufw allow from to any port 7008 +``` + +### **3. Authentication** + +**Enable simple auth:** +```yaml +DOZZLE_AUTH_PROVIDER: simple +``` + +สร้าง `01-infra/data/dozzle/users.yml`: +```yaml +users: + - name: admin + username: admin + password: $2a$10$... + email: admin@sriphat.com +``` + +### **4. Read-only Docker Socket** + +```yaml +volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro +``` + +## 📊 Monitoring + +### **Health Checks** + +```bash +# Main UI +curl http://localhost:9999/dozzle/healthcheck + +# Airbyte Agent +curl http://192.168.100.9:7007/healthcheck + +# Airflow Agent +curl http://192.168.100.9:7008/healthcheck +``` + +### **Logs** + +```bash +# Main UI +docker logs dozzle -f + +# Agents +docker logs dozzle-agent-airbyte -f +docker logs dozzle-agent-airflow -f +``` + +## 🎯 Best Practices + +1. **ใช้ Internal Network** - ไม่ expose agent ports ออกภายนอก +2. **Enable Authentication** - ใช้ simple auth หรือ forward proxy +3. **Monitor Agent Health** - ตั้ง healthcheck และ alerting +4. **Backup Configuration** - backup `users.yml` และ `.env` files +5. **Update Regularly** - อัพเดท Dozzle image เป็นประจำ +6. **Use HTTPS** - ใช้ SSL/TLS สำหรับ production +7. **Limit Access** - ใช้ firewall และ access lists + +## 📚 References + +- [Dozzle Documentation](https://dozzle.dev/) +- [Agent Mode Guide](https://dozzle.dev/guide/agent) +- [Authentication Guide](https://dozzle.dev/guide/authentication) +- [Remote Hosts Guide](https://dozzle.dev/guide/remote-hosts) + +## 🔄 Maintenance + +### **Update Dozzle** + +```bash +# Main UI +cd 01-infra +docker compose pull dozzle +docker compose up -d dozzle + +# Agents +docker pull amir20/dozzle:latest +docker restart dozzle-agent-airbyte +docker restart dozzle-agent-airflow +``` + +### **Backup Configuration** + +```bash +# Backup .env +cp .env.global .env.global.backup + +# Backup users.yml (if using auth) +cp 01-infra/data/dozzle/users.yml users.yml.backup +``` + +## 🎉 Summary + +**ตอนนี้คุณมี:** +- ✅ Dozzle Main UI บน Main Server +- ✅ Dozzle Agent บน Airbyte Host (192.168.100.9:7007) +- ✅ Dozzle Agent บน Airflow Host (192.168.100.9:7008) +- ✅ Nginx reverse proxy สำหรับ `/dozzle` subpath +- ✅ Multi-host monitoring ผ่าน single UI +- ✅ Real-time log streaming จากทุก hosts + +**เข้าใช้งานที่:** +``` +http://ai.sriphat.com/dozzle +``` + +**Features:** +- Monitor logs จาก Main Server, Airbyte, และ Airflow +- Real-time streaming +- Container stats +- Interactive shell +- Search และ filter