update config for limit resouce size
This commit is contained in:
jigoong
13
.env.global
13
.env.global
@@ -4,6 +4,7 @@ TZ=Asia/Bangkok
|
||||
|
||||
DB_HOST=postgres
|
||||
DB_PORT=5432
|
||||
DB_PORT_EXPOSE=5435
|
||||
DB_USER=postgres
|
||||
DB_PASSWORD=Secure_Hospital_Pass_2026
|
||||
DB_NAME=postgres
|
||||
@@ -13,6 +14,7 @@ POSTGRES_PASSWORD=Secure_Hospital_Pass_2026
|
||||
|
||||
KEYCLOAK_ADMIN=admin
|
||||
KEYCLOAK_ADMIN_PASSWORD=admin_secret_pass_2026
|
||||
KEYCLOAK_DB_NAME=keycloak
|
||||
|
||||
SUPERSET_SECRET_KEY=superset_random_secret_key_change_me_2026
|
||||
SUPERSET_ADMIN_USERNAME=admin
|
||||
@@ -29,3 +31,14 @@ AIRBYTE_PORT=8030
|
||||
AIRBYTE_BASIC_AUTH_USERNAME=
|
||||
AIRBYTE_BASIC_AUTH_PASSWORD=
|
||||
AIRBYTE_BASIC_AUTH_PROXY_TIMEOUT=900
|
||||
|
||||
# Dozzle - Docker Log Viewer & Monitoring
|
||||
DOZZLE_PORT=9999
|
||||
DOZZLE_LEVEL=info
|
||||
DOZZLE_BASE=/dozzle
|
||||
DOZZLE_HOSTNAME=Sriphat Main Server
|
||||
DOZZLE_AUTH_PROVIDER=none
|
||||
DOZZLE_RESTART_POLICY=unless-stopped
|
||||
# Remote agents: Airbyte and Airflow on 192.168.100.9
|
||||
# Format: host:port,host:port (comma-separated)
|
||||
DOZZLE_REMOTE_AGENT=192.168.100.9:7007
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -12,3 +12,5 @@ ruff_cache/
|
||||
*/data/
|
||||
01-infra/letsencrypt/
|
||||
.windsurf/
|
||||
_daily-log/
|
||||
daily-log/
|
||||
|
||||
10
03-apiservice-v0.1/.gitignore
vendored
10
03-apiservice-v0.1/.gitignore
vendored
@@ -1,10 +0,0 @@
|
||||
.env
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.venv/
|
||||
venv/
|
||||
.python-version
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
ruff_cache/
|
||||
.windsurf/
|
||||
@@ -1,17 +0,0 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY ./app /app/app
|
||||
|
||||
ENV TZ=Asia/Bangkok
|
||||
|
||||
EXPOSE 8040
|
||||
|
||||
CMD ["gunicorn","-k","uvicorn.workers.UvicornWorker","app.main:app","--bind","0.0.0.0:8040","--workers","2","--access-logfile","-","--error-logfile","-"]
|
||||
@@ -1,44 +0,0 @@
|
||||
# 03-apiservice: Custom FastAPI Service
|
||||
|
||||
## Build & Start
|
||||
```bash
|
||||
docker compose --env-file ../.env.global up --build -d
|
||||
```
|
||||
|
||||
## Access
|
||||
Internal only - access via Nginx Proxy Manager at `/apiservice`
|
||||
|
||||
## Admin UI
|
||||
- Login: http://<domain>/apiservice/admin/
|
||||
- Generate API Key: POST /apiservice/admin/api-keys/generate
|
||||
|
||||
## env
|
||||
env that important for provision
|
||||
```
|
||||
## supabase
|
||||
SUPABASE_DB_HOST=sdp-db
|
||||
SUPABASE_DB_PORT=5432
|
||||
SUPABASE_DB_USER=postgres.1
|
||||
SUPABASE_DB_PASSWORD=
|
||||
SUPABASE_DB_NAME=postgres
|
||||
SUPABASE_DB_SSLMODE=disable
|
||||
|
||||
## pgsql
|
||||
DB_HOST=postgres
|
||||
DB_PORT=5432
|
||||
DB_USER=postgres
|
||||
DB_PASSWORD=
|
||||
DB_NAME=postgres
|
||||
DB_SSLMODE=disable
|
||||
AIRBYTE_DB_NAME=airbyte
|
||||
KEYCLOAK_DB_NAME=keycloack
|
||||
SUPERSET_DB_NAME=superset
|
||||
#TEMPORAL_DB_NAME=temporal
|
||||
|
||||
## api
|
||||
ROOT_PATH=/apiservice
|
||||
APP_NAME=APIsService
|
||||
ADMIN_SECRET_KEY=
|
||||
ADMIN_USERNAME=admin
|
||||
ADMIN_PASSWORD=
|
||||
```
|
||||
@@ -1,283 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import HTTPException, Request, status
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqladmin import Admin, ModelView
|
||||
from sqladmin.authentication import AuthenticationBackend
|
||||
from starlette.responses import HTMLResponse, RedirectResponse
|
||||
from starlette.datastructures import URL
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from wtforms import BooleanField, SelectField, StringField
|
||||
from wtforms.validators import Optional
|
||||
|
||||
from app.core.config import settings
|
||||
from app.db.engine import engine
|
||||
from app.db.models import ApiClient, ApiKey
|
||||
from app.security.api_key import generate_api_key, get_prefix, hash_api_key
|
||||
|
||||
|
||||
class AdminAuth(AuthenticationBackend):
|
||||
async def login(self, request: Request) -> bool:
|
||||
form = await request.form()
|
||||
username = form.get("username")
|
||||
password = form.get("password")
|
||||
|
||||
if username == settings.ADMIN_USERNAME and password == settings.ADMIN_PASSWORD:
|
||||
request.session.update({"admin": True})
|
||||
return True
|
||||
return False
|
||||
|
||||
async def logout(self, request: Request) -> bool:
|
||||
request.session.clear()
|
||||
return True
|
||||
|
||||
async def authenticate(self, request: Request) -> bool:
|
||||
return bool(request.session.get("admin"))
|
||||
|
||||
|
||||
class ApiClientAdmin(ModelView, model=ApiClient):
|
||||
column_list = [ApiClient.id, ApiClient.name, ApiClient.is_active]
|
||||
|
||||
async def insert_model(self, request: Request, data: dict) -> ApiClient:
|
||||
obj: ApiClient = await super().insert_model(request, data)
|
||||
|
||||
plain_key = generate_api_key()
|
||||
|
||||
db = sessionmaker(bind=engine, autoflush=False, autocommit=False)()
|
||||
try:
|
||||
api_key = ApiKey(
|
||||
client_id=obj.id,
|
||||
name="auto",
|
||||
key_prefix=get_prefix(plain_key),
|
||||
key_hash=hash_api_key(plain_key),
|
||||
permissions=[],
|
||||
is_active=True,
|
||||
)
|
||||
db.add(api_key)
|
||||
db.commit()
|
||||
db.refresh(api_key)
|
||||
|
||||
request.session["generated_api_key"] = {
|
||||
"client_id": obj.id,
|
||||
"client_name": obj.name,
|
||||
"key_id": api_key.id,
|
||||
"api_key": plain_key,
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return obj
|
||||
|
||||
|
||||
class ApiKeyAdmin(ModelView, model=ApiKey):
|
||||
column_list = [ApiKey.id, ApiKey.client_id, ApiKey.name, ApiKey.is_active, ApiKey.permissions]
|
||||
form_excluded_columns = [ApiKey.key_hash, ApiKey.key_prefix, ApiKey.created_at]
|
||||
|
||||
form_extra_fields = {
|
||||
"plain_key": StringField("Plain Key", validators=[Optional()]),
|
||||
"permissions_csv": StringField("Permissions (comma)", validators=[Optional()]),
|
||||
"endpoint_path": SelectField("Endpoint", choices=[], validators=[Optional()]),
|
||||
"perm_read": BooleanField("Read (GET)"),
|
||||
"perm_write": BooleanField("Write (POST/PATCH)"),
|
||||
"perm_delete": BooleanField("Delete (DELETE)"),
|
||||
}
|
||||
|
||||
async def on_model_change(self, data: dict, model: ApiKey, is_created: bool, request: Request) -> None:
|
||||
plain_key = data.get("plain_key")
|
||||
if not plain_key and is_created:
|
||||
plain_key = generate_api_key()
|
||||
|
||||
if plain_key:
|
||||
model.key_prefix = get_prefix(plain_key)
|
||||
model.key_hash = hash_api_key(plain_key)
|
||||
|
||||
if is_created:
|
||||
request.state.generated_api_key_plain = plain_key
|
||||
|
||||
permissions: list[str] = []
|
||||
endpoint_path = data.get("endpoint_path")
|
||||
if endpoint_path:
|
||||
if data.get("perm_read"):
|
||||
permissions.append(f"{endpoint_path}:read")
|
||||
if data.get("perm_write"):
|
||||
permissions.append(f"{endpoint_path}:write")
|
||||
if data.get("perm_delete"):
|
||||
permissions.append(f"{endpoint_path}:delete")
|
||||
|
||||
permissions_csv = data.get("permissions_csv")
|
||||
if permissions_csv is not None:
|
||||
perms = [p.strip() for p in permissions_csv.split(",") if p.strip()]
|
||||
permissions.extend(perms)
|
||||
|
||||
if permissions:
|
||||
seen: set[str] = set()
|
||||
deduped: list[str] = []
|
||||
for p in permissions:
|
||||
if p not in seen:
|
||||
seen.add(p)
|
||||
deduped.append(p)
|
||||
model.permissions = deduped
|
||||
|
||||
async def after_model_change(self, data: dict, model: ApiKey, is_created: bool, request: Request) -> None:
|
||||
if not is_created:
|
||||
return
|
||||
|
||||
plain_key = getattr(request.state, "generated_api_key_plain", None)
|
||||
if not plain_key:
|
||||
return
|
||||
|
||||
request.session["generated_api_key"] = {
|
||||
"client_id": model.client_id,
|
||||
"client_name": str(getattr(model, "client", "")) if getattr(model, "client", None) else "",
|
||||
"key_id": model.id,
|
||||
"api_key": plain_key,
|
||||
}
|
||||
|
||||
|
||||
def mount_admin(app):
|
||||
auth_backend = AdminAuth(secret_key=settings.ADMIN_SECRET_KEY)
|
||||
|
||||
class CustomAdmin(Admin):
|
||||
def get_save_redirect_url(
|
||||
self, request: Request, form, model_view: ModelView, obj
|
||||
):
|
||||
if (
|
||||
getattr(model_view, "model", None) in (ApiClient, ApiKey)
|
||||
and request.session.get("generated_api_key")
|
||||
):
|
||||
root_path = request.scope.get("root_path") or ""
|
||||
return URL(f"{root_path}/admin/generated-api-key")
|
||||
|
||||
return super().get_save_redirect_url(
|
||||
request=request,
|
||||
form=form,
|
||||
model_view=model_view,
|
||||
obj=obj,
|
||||
)
|
||||
|
||||
admin = CustomAdmin(
|
||||
app=app,
|
||||
engine=engine,
|
||||
authentication_backend=auth_backend,
|
||||
title="My Service Management",
|
||||
base_url="/admin",
|
||||
)
|
||||
|
||||
openapi = app.openapi()
|
||||
paths = openapi.get("paths") or {}
|
||||
endpoint_choices: list[tuple[str, str]] = []
|
||||
for path in sorted(paths.keys()):
|
||||
if not path.startswith("/api/"):
|
||||
continue
|
||||
methods = paths.get(path) or {}
|
||||
available = sorted([m.upper() for m in methods.keys()])
|
||||
label = f"{path} [{' '.join(available)}]" if available else path
|
||||
endpoint_choices.append((path, label))
|
||||
ApiKeyAdmin.form_extra_fields["endpoint_path"].kwargs["choices"] = endpoint_choices
|
||||
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
admin.add_view(ApiClientAdmin)
|
||||
admin.add_view(ApiKeyAdmin)
|
||||
|
||||
@app.get("/admin/generated-api-key")
|
||||
async def _admin_generated_api_key(request: Request):
|
||||
if not request.session.get("admin"):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
|
||||
key_info = request.session.pop("generated_api_key", None)
|
||||
root_path = request.scope.get("root_path") or ""
|
||||
clients_url = f"{root_path}/admin/{ApiClientAdmin.identity}/list"
|
||||
|
||||
if not key_info:
|
||||
return HTMLResponse(
|
||||
f"<h2>No API key to display</h2><p>The API key was already shown or expired.</p><p><a href=\"{clients_url}\">Back to clients</a></p>",
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
client_name = key_info.get("client_name", "")
|
||||
client_id = key_info.get("client_id", "")
|
||||
key_id = key_info.get("key_id", "")
|
||||
api_key = key_info.get("api_key", "")
|
||||
|
||||
return HTMLResponse(
|
||||
(
|
||||
"<h2>API key generated</h2>"
|
||||
"<p>Copy this API key now. You won't be able to view it again.</p>"
|
||||
f"<p><b>Client</b>: {client_name} (ID: {client_id})</p>"
|
||||
f"<p><b>Key ID</b>: {key_id}</p>"
|
||||
f"<pre style=\"padding:12px;border:1px solid #ddd;background:#f7f7f7;\">{api_key}</pre>"
|
||||
f"<p><a href=\"{clients_url}\">Back to clients</a></p>"
|
||||
),
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
@app.get("/admin/clients/{client_id}/generate-api-key")
|
||||
async def _admin_generate_api_key_get(
|
||||
request: Request,
|
||||
client_id: int,
|
||||
permissions: str = "",
|
||||
name: str | None = None,
|
||||
):
|
||||
if not request.session.get("admin"):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
|
||||
perms = [p.strip() for p in permissions.split(",") if p.strip()]
|
||||
plain_key = generate_api_key()
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
client = db.get(ApiClient, client_id)
|
||||
if not client:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Client not found")
|
||||
|
||||
api_key = ApiKey(
|
||||
client_id=client_id,
|
||||
name=name,
|
||||
key_prefix=get_prefix(plain_key),
|
||||
key_hash=hash_api_key(plain_key),
|
||||
permissions=perms,
|
||||
is_active=True,
|
||||
)
|
||||
db.add(api_key)
|
||||
db.commit()
|
||||
db.refresh(api_key)
|
||||
|
||||
return {"key_id": api_key.id, "api_key": plain_key, "permissions": perms}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@app.post("/admin/api-keys/generate")
|
||||
async def _admin_generate_api_key(
|
||||
request: Request,
|
||||
client_id: int,
|
||||
permissions: str = "",
|
||||
name: str | None = None,
|
||||
):
|
||||
if not request.session.get("admin"):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated")
|
||||
|
||||
perms = [p.strip() for p in permissions.split(",") if p.strip()]
|
||||
plain_key = generate_api_key()
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
client = db.get(ApiClient, client_id)
|
||||
if not client:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Client not found")
|
||||
|
||||
api_key = ApiKey(
|
||||
client_id=client_id,
|
||||
name=name,
|
||||
key_prefix=get_prefix(plain_key),
|
||||
key_hash=hash_api_key(plain_key),
|
||||
permissions=perms,
|
||||
is_active=True,
|
||||
)
|
||||
db.add(api_key)
|
||||
db.commit()
|
||||
db.refresh(api_key)
|
||||
|
||||
return {"key_id": api_key.id, "api_key": plain_key, "permissions": perms}
|
||||
finally:
|
||||
db.close()
|
||||
@@ -1,127 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.v1.schemas import FeedCheckpointIn
|
||||
from app.core.config import settings
|
||||
from app.db.models import RawOpdCheckpoint
|
||||
from app.security.dependencies import get_db, get_supabase_db, require_permission
|
||||
from app.utils.supabase_client import SupabaseAPIError, upsert_to_supabase_sync
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
|
||||
PERM_FEED_CHECKPOINT_WRITE = "/api/v1/feed/checkpoint:write"
|
||||
PERM_FEED_CHECKPOINT_WRITE_LEGACY = "feed.checkpoint:write"
|
||||
|
||||
|
||||
def _to_tz(dt):
|
||||
if dt is None:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
return dt.replace(tzinfo=ZoneInfo(settings.TIMEZONE))
|
||||
return dt.astimezone(ZoneInfo(settings.TIMEZONE))
|
||||
|
||||
|
||||
def _to_iso(dt):
|
||||
"""Convert datetime to ISO 8601 string for Supabase API."""
|
||||
if dt is None:
|
||||
return None
|
||||
return dt.isoformat()
|
||||
|
||||
|
||||
@router.post("/feed/checkpoint")
|
||||
def upsert_feed_checkpoint(
|
||||
payload: list[FeedCheckpointIn],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
):
|
||||
rows = []
|
||||
supabase_rows = []
|
||||
|
||||
#clean_data = payload.model_dump(exclude_none=True)
|
||||
for item in payload:
|
||||
# Prepare data for local database 'default' if item.id is None else
|
||||
row = {
|
||||
"id": item.id,
|
||||
"hn": item.hn,
|
||||
"vn": item.vn,
|
||||
"location": item.location,
|
||||
"type": item.type,
|
||||
"timestamp_in": _to_tz(item.timestamp_in),
|
||||
"timestamp_out": _to_tz(item.timestamp_out),
|
||||
"waiting_time": item.waiting_time,
|
||||
"bu": item.bu,
|
||||
}
|
||||
if item.id is None:
|
||||
del(row["id"])
|
||||
rows.append(row)
|
||||
|
||||
# Prepare data for Supabase API (convert datetime to ISO string) 'default' if item.id is None else
|
||||
supabase_row = {
|
||||
"id": item.id,
|
||||
"hn": item.hn,
|
||||
"vn": item.vn,
|
||||
"location": item.location,
|
||||
"type": item.type,
|
||||
"timestamp_in": _to_iso(_to_tz(item.timestamp_in)),
|
||||
"timestamp_out": _to_iso(_to_tz(item.timestamp_out)),
|
||||
"waiting_time": item.waiting_time,
|
||||
"bu": item.bu,
|
||||
}
|
||||
if item.id is None:
|
||||
del(supabase_row["id"])
|
||||
supabase_rows.append(supabase_row)
|
||||
|
||||
# Insert/update to local database
|
||||
stmt = insert(RawOpdCheckpoint).values(rows)
|
||||
update_cols = {
|
||||
"id": stmt.excluded.id,
|
||||
"type": stmt.excluded.type,
|
||||
"timestamp_in": stmt.excluded.timestamp_in,
|
||||
"timestamp_out": stmt.excluded.timestamp_out,
|
||||
"waiting_time": stmt.excluded.waiting_time,
|
||||
"bu": stmt.excluded.bu,
|
||||
}
|
||||
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=[RawOpdCheckpoint.hn, RawOpdCheckpoint.vn, RawOpdCheckpoint.location, RawOpdCheckpoint.timestamp_in],
|
||||
set_=update_cols,
|
||||
)
|
||||
result = db.execute(stmt)
|
||||
db.commit()
|
||||
|
||||
# Send data to Supabase via API call
|
||||
supabase_result = None
|
||||
supabase_error = None
|
||||
|
||||
try:
|
||||
logger.info(f"Sending {len(supabase_rows)} records to Supabase API")
|
||||
supabase_result = upsert_to_supabase_sync(
|
||||
table="raw_opd_checkpoint",
|
||||
data=supabase_rows,
|
||||
on_conflict="hn,vn,location,timestamp_in",
|
||||
)
|
||||
logger.info(f"Successfully sent data to Supabase: {supabase_result.get('status_code')}")
|
||||
except SupabaseAPIError as e:
|
||||
logger.error(f"Failed to send data to Supabase: {str(e)}")
|
||||
supabase_error = str(e)
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error sending data to Supabase: {str(e)}")
|
||||
supabase_error = f"Unexpected error: {str(e)}"
|
||||
|
||||
return {
|
||||
"upserted": len(rows),
|
||||
"rowcount": result.rowcount,
|
||||
"supabase": {
|
||||
"success": supabase_result is not None,
|
||||
"result": supabase_result,
|
||||
"error": supabase_error,
|
||||
},
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class FeedCheckpointIn(BaseModel):
|
||||
id: int | None = None
|
||||
hn: int
|
||||
vn: int
|
||||
location: str
|
||||
type: str
|
||||
timestamp_in: datetime
|
||||
timestamp_out: datetime | None = None
|
||||
waiting_time: int | None = None
|
||||
bu: str | None = None
|
||||
@@ -1,35 +0,0 @@
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
APP_NAME: str = "APIsService"
|
||||
|
||||
DB_HOST: str
|
||||
DB_PORT: int = 5432
|
||||
DB_USER: str
|
||||
DB_PASSWORD: str
|
||||
DB_NAME: str
|
||||
DB_SSLMODE: str = "disable"
|
||||
|
||||
SUPABASE_DB_HOST: str
|
||||
SUPABASE_DB_PORT: int = 5432
|
||||
SUPABASE_DB_USER: str
|
||||
SUPABASE_DB_PASSWORD: str
|
||||
SUPABASE_DB_NAME: str
|
||||
SUPABASE_DB_SSLMODE: str = "disable"
|
||||
|
||||
SUPABASE_API_URL: str
|
||||
SUPABASE_API_KEY: str
|
||||
|
||||
ROOT_PATH: str = ""
|
||||
|
||||
TIMEZONE: str = "Asia/Bangkok"
|
||||
|
||||
ADMIN_SECRET_KEY: str
|
||||
ADMIN_USERNAME: str
|
||||
ADMIN_PASSWORD: str
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -1,5 +0,0 @@
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
@@ -1,35 +0,0 @@
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
def build_db_url() -> str:
|
||||
user = quote_plus(settings.DB_USER)
|
||||
password = quote_plus(settings.DB_PASSWORD)
|
||||
host = settings.DB_HOST
|
||||
port = settings.DB_PORT
|
||||
db = quote_plus(settings.DB_NAME)
|
||||
|
||||
return (
|
||||
f"postgresql+psycopg://{user}:{password}@{host}:{port}/{db}"
|
||||
f"?sslmode={quote_plus(settings.DB_SSLMODE)}"
|
||||
)
|
||||
|
||||
|
||||
def build_supabase_db_url() -> str:
|
||||
user = quote_plus(settings.SUPABASE_DB_USER)
|
||||
password = quote_plus(settings.SUPABASE_DB_PASSWORD)
|
||||
host = settings.SUPABASE_DB_HOST
|
||||
port = settings.SUPABASE_DB_PORT
|
||||
db = quote_plus(settings.SUPABASE_DB_NAME)
|
||||
|
||||
return (
|
||||
f"postgresql+psycopg://{user}:{password}@{host}:{port}/{db}"
|
||||
f"?sslmode={quote_plus(settings.SUPABASE_DB_SSLMODE)}"
|
||||
)
|
||||
|
||||
|
||||
engine = create_engine(build_db_url(), pool_pre_ping=True)
|
||||
supabase_engine = create_engine(build_supabase_db_url(), pool_pre_ping=True)
|
||||
@@ -1,13 +0,0 @@
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db.base import Base
|
||||
from app.db.engine import engine
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
# with engine.begin() as conn:
|
||||
# conn.execute(text("CREATE SCHEMA IF NOT EXISTS fastapi"))
|
||||
# conn.execute(text("CREATE SCHEMA IF NOT EXISTS operationbi"))
|
||||
|
||||
# Base.metadata.create_all(bind=conn)
|
||||
pass
|
||||
@@ -1,74 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.db.base import Base
|
||||
|
||||
|
||||
class RawOpdCheckpoint(Base):
|
||||
__tablename__ = "raw_opd_checkpoint"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("hn", "vn", "location", name="uq_raw_opd_checkpoint_hn_vn_location"),
|
||||
{"schema": "rawdata"},
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
hn: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
vn: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
location: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
timestamp_in: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
timestamp_out: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
waiting_time: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
bu: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
|
||||
class ApiClient(Base):
|
||||
__tablename__ = "api_client"
|
||||
__table_args__ = {"schema": "fastapi"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
name: Mapped[str] = mapped_column(String(128), unique=True, nullable=False)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
|
||||
api_keys: Mapped[list[ApiKey]] = relationship(
|
||||
back_populates="client",
|
||||
cascade="all, delete-orphan",
|
||||
passive_deletes=True,
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
client_id = getattr(self, "id", None)
|
||||
if client_id is None:
|
||||
return self.name
|
||||
return f"{self.name} ({client_id})"
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return str(self)
|
||||
|
||||
|
||||
class ApiKey(Base):
|
||||
__tablename__ = "api_key"
|
||||
__table_args__ = {"schema": "fastapi"}
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
client_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("fastapi.api_client.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
name: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
key_prefix: Mapped[str] = mapped_column(String(12), nullable=False)
|
||||
key_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
|
||||
permissions: Mapped[list[str]] = mapped_column(JSONB, nullable=False, default=list)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
client: Mapped[ApiClient] = relationship(back_populates="api_keys")
|
||||
@@ -1,99 +0,0 @@
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from starlette.datastructures import Headers
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
class ForceHTTPSMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request, call_next):
|
||||
# บังคับให้ FastAPI มองว่า Request ที่เข้ามาเป็น HTTPS เสมอ
|
||||
# เพื่อให้ url_for() เจนลิงก์ CSS/JS เป็น https://
|
||||
request.scope["scheme"] = "https"
|
||||
response = await call_next(request)
|
||||
return response
|
||||
|
||||
|
||||
class ForwardedProtoMiddleware:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] in {"http", "websocket"}:
|
||||
headers = Headers(scope=scope)
|
||||
forwarded_proto = headers.get("x-forwarded-proto")
|
||||
if forwarded_proto:
|
||||
proto = forwarded_proto.split(",", 1)[0].strip()
|
||||
if proto:
|
||||
new_scope = dict(scope)
|
||||
new_scope["scheme"] = proto
|
||||
return await self.app(new_scope, receive, send)
|
||||
|
||||
return await self.app(scope, receive, send)
|
||||
|
||||
|
||||
# class RootPathStripMiddleware:
|
||||
# def __init__(self, app, prefix: str):
|
||||
# self.app = app
|
||||
# self.prefix = (prefix or "").rstrip("/")
|
||||
|
||||
# async def __call__(self, scope, receive, send):
|
||||
# if scope["type"] in {"http", "websocket"} and self.prefix:
|
||||
# path = scope.get("path") or ""
|
||||
# new_scope = dict(scope)
|
||||
# new_scope["root_path"] = self.prefix
|
||||
|
||||
# if path == self.prefix or path.startswith(self.prefix + "/"):
|
||||
# new_path = path[len(self.prefix) :]
|
||||
# new_scope["path"] = new_path if new_path else "/"
|
||||
|
||||
# return await self.app(new_scope, receive, send)
|
||||
|
||||
# return await self.app(scope, receive, send)
|
||||
|
||||
from app.admin import mount_admin
|
||||
from app.api.v1.routes import router as v1_router
|
||||
from app.core.config import settings
|
||||
from app.db.init_db import init_db
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqladmin import Admin
|
||||
import os
|
||||
import sqladmin
|
||||
|
||||
# รายชื่อ Origins ที่อนุญาตให้ยิง API มาหาเราได้
|
||||
origins = [
|
||||
"http://localhost:80400", # สำหรับตอนพัฒนา Frontend
|
||||
"https://ai.sriphat.com", # Domain หลักของคุณ
|
||||
"http://ai.sriphat.com",
|
||||
]
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
init_db()
|
||||
yield
|
||||
|
||||
print(settings.ROOT_PATH, flush=True)
|
||||
|
||||
sqladmin_dir = os.path.dirname(sqladmin.__file__)
|
||||
statics_path = os.path.join(sqladmin_dir, "statics")
|
||||
|
||||
app = FastAPI(title=settings.APP_NAME, root_path=settings.ROOT_PATH, lifespan=lifespan)
|
||||
#if settings.ROOT_PATH:
|
||||
# app.add_middleware(RootPathStripMiddleware, prefix=settings.ROOT_PATH)
|
||||
app.add_middleware(ForceHTTPSMiddleware)
|
||||
app.add_middleware(SessionMiddleware, secret_key=settings.ADMIN_SECRET_KEY)
|
||||
app.add_middleware(ForwardedProtoMiddleware)
|
||||
app.include_router(v1_router)
|
||||
app.mount("/admin/statics", StaticFiles(directory=statics_path), name="admin_statics")
|
||||
app.mount("/apiservice/admin/statics", StaticFiles(directory=statics_path), name="proxy_admin_statics")
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=origins, # หรือ ["*"] ถ้าต้องการอนุญาตทั้งหมด (ไม่แนะนำใน production)
|
||||
allow_credentials=True, # สำคัญมาก! ต้องเป็น True ถ้าหน้า Admin/API มีการใช้ Cookies/Sessions
|
||||
allow_methods=["*"], # อนุญาตทุก HTTP Method (GET, POST, PUT, DELETE, etc.)
|
||||
allow_headers=["*"], # อนุญาตทุก Headers
|
||||
)
|
||||
|
||||
mount_admin(app)
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
import secrets
|
||||
|
||||
import bcrypt
|
||||
|
||||
|
||||
def generate_api_key(prefix_len: int = 8, token_bytes: int = 32) -> str:
|
||||
prefix = secrets.token_urlsafe(prefix_len)[:prefix_len]
|
||||
token = secrets.token_urlsafe(token_bytes)
|
||||
return f"{prefix}.{token}"
|
||||
|
||||
|
||||
def get_prefix(api_key: str) -> str:
|
||||
return api_key.split(".", 1)[0]
|
||||
|
||||
|
||||
def hash_api_key(api_key: str) -> str:
|
||||
hashed = bcrypt.hashpw(api_key.encode("utf-8"), bcrypt.gensalt())
|
||||
return hashed.decode("utf-8")
|
||||
|
||||
|
||||
def verify_api_key(api_key: str, api_key_hash: str) -> bool:
|
||||
return bcrypt.checkpw(api_key.encode("utf-8"), api_key_hash.encode("utf-8"))
|
||||
@@ -1,66 +0,0 @@
|
||||
from typing import Annotated
|
||||
from collections.abc import Sequence
|
||||
|
||||
from fastapi import Depends, HTTPException, Request, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.db.engine import engine, supabase_engine
|
||||
from app.db.models import ApiKey
|
||||
from app.security.api_key import get_prefix, verify_api_key
|
||||
|
||||
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
SupabaseSessionLocal = sessionmaker(bind=supabase_engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def get_supabase_db():
|
||||
db = SupabaseSessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def get_bearer_token(request: Request) -> str:
|
||||
auth = request.headers.get("authorization")
|
||||
if not auth:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Missing Authorization")
|
||||
|
||||
parts = auth.split(" ", 1)
|
||||
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip():
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid Authorization")
|
||||
|
||||
return parts[1].strip()
|
||||
|
||||
|
||||
def require_permission(permission: str | Sequence[str]):
|
||||
def _dep(
|
||||
token: Annotated[str, Depends(get_bearer_token)],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> ApiKey:
|
||||
prefix = get_prefix(token)
|
||||
stmt = select(ApiKey).where(ApiKey.key_prefix == prefix, ApiKey.is_active.is_(True))
|
||||
api_key = db.execute(stmt).scalar_one_or_none()
|
||||
if not api_key:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key")
|
||||
|
||||
if not verify_api_key(token, api_key.key_hash):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key")
|
||||
|
||||
allowed = set(api_key.permissions or [])
|
||||
required = [permission] if isinstance(permission, str) else list(permission)
|
||||
if not any(p in allowed for p in required):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Permission denied")
|
||||
|
||||
return api_key
|
||||
|
||||
return _dep
|
||||
@@ -1,37 +0,0 @@
|
||||
services:
|
||||
apiservice:
|
||||
build: .
|
||||
container_name: apiservice
|
||||
env_file:
|
||||
- ../.env.global
|
||||
environment:
|
||||
- TZ=${TZ:-Asia/Bangkok}
|
||||
- DB_HOST=${DB_HOST}
|
||||
- DB_PORT=${DB_PORT}
|
||||
- DB_USER=${DB_USER}
|
||||
- DB_PASSWORD=${DB_PASSWORD}
|
||||
- DB_NAME=${DB_NAME}
|
||||
- DB_SSLMODE=${DB_SSLMODE}
|
||||
- ROOT_PATH=${ROOT_PATH}
|
||||
- APP_NAME=${APP_NAME}
|
||||
- ADMIN_SECRET_KEY=${ADMIN_SECRET_KEY}
|
||||
- ADMIN_USERNAME=${ADMIN_USERNAME}
|
||||
- ADMIN_PASSWORD=${ADMIN_PASSWORD}
|
||||
networks:
|
||||
- shared_data_network
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./app:/app/app
|
||||
- .env:/app/.env
|
||||
ports:
|
||||
- 0.0.0.0:8040:8040
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8040/apiservice/docs', timeout=5).read()"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
|
||||
networks:
|
||||
shared_data_network:
|
||||
external: true
|
||||
@@ -1,16 +0,0 @@
|
||||
fastapi==0.115.8
|
||||
uvicorn==0.34.0
|
||||
gunicorn==23.0.0
|
||||
SQLAlchemy==2.0.38
|
||||
psycopg==3.2.5
|
||||
pydantic==2.10.6
|
||||
pydantic-settings==2.7.1
|
||||
psycopg[binary]
|
||||
sqladmin==0.20.1
|
||||
itsdangerous==2.2.0
|
||||
bcrypt==4.3.0
|
||||
python-multipart==0.0.20
|
||||
httpx==0.28.1
|
||||
WTForms
|
||||
#==3.2.1
|
||||
|
||||
@@ -40,7 +40,7 @@ might_contain_dag_callable = airflow.utils.file.might_contain_dag_via_default_he
|
||||
#
|
||||
# Variable: AIRFLOW__CORE__DEFAULT_TIMEZONE
|
||||
#
|
||||
default_timezone = utc
|
||||
default_timezone = Asia/Bangkok
|
||||
|
||||
# The executor class that airflow should use. Choices include
|
||||
# ``LocalExecutor``, ``CeleryExecutor``,
|
||||
@@ -90,7 +90,7 @@ simple_auth_manager_all_admins = False
|
||||
#
|
||||
# Variable: AIRFLOW__CORE__PARALLELISM
|
||||
#
|
||||
parallelism = 8
|
||||
parallelism = 2
|
||||
|
||||
# The maximum number of task instances allowed to run concurrently in each dag run.
|
||||
# This is also configurable per-dag with ``max_active_tasks``,
|
||||
@@ -115,7 +115,7 @@ dags_are_paused_at_creation = True
|
||||
#
|
||||
# Variable: AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG
|
||||
#
|
||||
max_active_runs_per_dag = 16
|
||||
max_active_runs_per_dag = 1
|
||||
|
||||
# (experimental) The maximum number of consecutive DAG failures before DAG is automatically paused.
|
||||
# This is also configurable per DAG level with ``max_consecutive_failed_dag_runs``,
|
||||
@@ -2166,7 +2166,7 @@ refresh_interval = 300
|
||||
#
|
||||
# Variable: AIRFLOW__DAG_PROCESSOR__PARSING_PROCESSES
|
||||
#
|
||||
parsing_processes = 2
|
||||
parsing_processes = 1
|
||||
|
||||
# One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``.
|
||||
# The DAG processor will list and sort the dag files to decide the parsing order.
|
||||
@@ -2193,7 +2193,9 @@ max_callbacks_per_loop = 20
|
||||
#
|
||||
# Variable: AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL
|
||||
#
|
||||
min_file_process_interval = 30
|
||||
min_file_process_interval = 90
|
||||
|
||||
dag_dir_list_interval = 90
|
||||
|
||||
# How long (in seconds) to wait after we have re-parsed a DAG file before deactivating stale
|
||||
# DAGs (DAGs which are no longer present in the expected files). The reason why we need
|
||||
@@ -2491,7 +2493,7 @@ flower_basic_auth =
|
||||
#
|
||||
# Variable: AIRFLOW__CELERY__SYNC_PARALLELISM
|
||||
#
|
||||
sync_parallelism = 0
|
||||
sync_parallelism = 2
|
||||
|
||||
# Import path for celery configuration options
|
||||
#
|
||||
|
||||
@@ -64,7 +64,7 @@ x-airflow-common:
|
||||
AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
|
||||
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY:-}
|
||||
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
|
||||
AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES:-'false'}
|
||||
AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES:-False}
|
||||
AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/'
|
||||
# yamllint disable rule:line-length
|
||||
# Use simple http server on scheduler for health checks
|
||||
@@ -76,18 +76,22 @@ x-airflow-common:
|
||||
_PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
|
||||
# The following line can be used to set a custom config file, stored in the local config folder
|
||||
AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'
|
||||
AIRFLOW__WEBSERVER__BASE_URL: ${AIRFLOW__WEBSERVER__BASE_URL:-https://ai.sriphat.com/airflow}
|
||||
AIRFLOW__API__BASE_URL: ${AIRFLOW__WEBSERVER__BASE_URL:-https://ai.sriphat.com/airflow}
|
||||
AIRFLOW__WEBSERVER__WEB_SERVER_PORT: ${AIRFLOW__WEBSERVER__WEB_SERVER_PORT:-8080}
|
||||
volumes:
|
||||
- ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags
|
||||
- ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
|
||||
- ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config
|
||||
- ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins
|
||||
user: "${AIRFLOW_UID:-50000}:0"
|
||||
depends_on:
|
||||
x-depends_on:
|
||||
&airflow-common-depends-on
|
||||
{}
|
||||
# airflow-base:
|
||||
# condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
# redis:
|
||||
# condition: service_healthy
|
||||
networks:
|
||||
- shared_data_network
|
||||
|
||||
@@ -114,19 +118,19 @@ services:
|
||||
# start_period: 5s
|
||||
# restart: always
|
||||
|
||||
redis:
|
||||
# Redis is limited to 7.2-bookworm due to licencing change
|
||||
# https://redis.io/blog/redis-adopts-dual-source-available-licensing/
|
||||
image: redis:7.2-bookworm
|
||||
expose:
|
||||
- 6379
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 30s
|
||||
retries: 50
|
||||
start_period: 30s
|
||||
restart: always
|
||||
# redis:
|
||||
# # Redis is limited to 7.2-bookworm due to licencing change
|
||||
# # https://redis.io/blog/redis-adopts-dual-source-available-licensing/
|
||||
# image: redis:7.2-bookworm
|
||||
# expose:
|
||||
# - 6379
|
||||
# healthcheck:
|
||||
# test: ["CMD", "redis-cli", "ping"]
|
||||
# interval: 10s
|
||||
# timeout: 30s
|
||||
# retries: 50
|
||||
# start_period: 30s
|
||||
# restart: always
|
||||
|
||||
airflow-apiserver:
|
||||
<<: *airflow-common
|
||||
|
||||
@@ -2,9 +2,58 @@ import os
|
||||
|
||||
SECRET_KEY = os.environ.get('SUPERSET_SECRET_KEY')
|
||||
SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg2://{os.environ.get('DATABASE_USER')}:{os.environ.get('DATABASE_PASSWORD')}@{os.environ.get('DATABASE_HOST')}:{os.environ.get('DATABASE_PORT')}/{os.environ.get('DATABASE_DB')}"
|
||||
|
||||
ENABLE_PROXY_FIX = True
|
||||
PUBLIC_ROLE_LIKE = "Gamma"
|
||||
|
||||
WTF_CSRF_ENABLED = True
|
||||
WTF_CSRF_ENABLED = False
|
||||
WTF_CSRF_TIME_LIMIT = None
|
||||
|
||||
FEATURE_FLAGS = {
|
||||
"EMBEDDED_SUPERSET": True,
|
||||
}
|
||||
|
||||
GUEST_ROLE_NAME = "Gamma"
|
||||
|
||||
ENABLE_CORS = True
|
||||
CORS_OPTIONS = {
|
||||
'supports_credentials': True,
|
||||
'allow_headers': ['*'],
|
||||
'resources': ['*'],
|
||||
'origins': ['*']
|
||||
}
|
||||
|
||||
SESSION_COOKIE_SAMESITE = "Lax"
|
||||
SESSION_COOKIE_SECURE = False
|
||||
|
||||
GUEST_TOKEN_JWT_SECRET = 'RgSCvATmH8fzluoFB6cqkdCXsY7jjq/zwGLRatoxYtI='
|
||||
GUEST_TOKEN_JWT_EXP_SECONDS = 86400 # 24 hours
|
||||
|
||||
# Logo link configuration
|
||||
LOGO_TARGET_PATH = '/superset/welcome/'
|
||||
|
||||
# Embedded SDK Configuration
|
||||
EMBEDDED_SUPERSET = True
|
||||
TALISMAN_ENABLED = False
|
||||
ENABLE_TEMPLATE_PROCESSING = True
|
||||
|
||||
# Guest token configuration for embedded SDK
|
||||
GUEST_TOKEN_JWT_ALGORITHM = "HS256"
|
||||
GUEST_TOKEN_JWT_EXP_SECONDS = 300 # 5 minutes
|
||||
|
||||
# Domain whitelist for embedded dashboards
|
||||
WEBDRIVER_BASEURL_USER_FRIENDLY_NAME = "Sriphat Dashboard"
|
||||
|
||||
|
||||
# Embedded SDK Domain Whitelist
|
||||
EMBEDDED_SDK_HOST_WHITELIST = [
|
||||
"http://localhost:8800",
|
||||
"https://ai.sriphat.com",
|
||||
"http://127.0.0.1:8800"
|
||||
]
|
||||
|
||||
# Allow embedding from specific domains
|
||||
TALISMAN_ALLOWED_DOMAINS = [
|
||||
"http://localhost:8800",
|
||||
"https://ai.sriphat.com",
|
||||
"http://127.0.0.1:8800"
|
||||
]
|
||||
|
||||
400
REMOTE_HOSTS_DOZZLE_SETUP.md
Normal file
400
REMOTE_HOSTS_DOZZLE_SETUP.md
Normal file
@@ -0,0 +1,400 @@
|
||||
# Dozzle Multi-Host Setup Guide
|
||||
|
||||
คู่มือการตั้งค่า Dozzle สำหรับ monitor Docker containers บนหลาย hosts
|
||||
|
||||
## 🏗️ Architecture Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Main Server (Current Host) │
|
||||
│ ├─ Nginx Proxy Manager │
|
||||
│ ├─ Keycloak │
|
||||
│ ├─ PostgreSQL │
|
||||
│ ├─ API Service │
|
||||
│ ├─ Supabase │
|
||||
│ ├─ Superset │
|
||||
│ └─ Dozzle (Main UI) ──────────────┐ │
|
||||
└───────────────────────────────────┼─────────────────────────┘
|
||||
│
|
||||
┌───────────────┴───────────────┐
|
||||
│ │
|
||||
┌───────────▼──────────┐ ┌────────────▼─────────┐
|
||||
│ 192.168.100.9 │ │ 192.168.100.9 │
|
||||
│ Airbyte Host │ │ Airflow Host │
|
||||
│ ├─ Airbyte Services │ │ ├─ Airflow Services │
|
||||
│ └─ Dozzle Agent │ │ └─ Dozzle Agent │
|
||||
│ (Port 7007) │ │ (Port 7008) │
|
||||
└──────────────────────┘ └──────────────────────┘
|
||||
```
|
||||
|
||||
## 📋 Setup Steps
|
||||
|
||||
### **Step 1: ติดตั้ง Dozzle Agent บน Remote Hosts**
|
||||
|
||||
#### **สำหรับ Airbyte Host (192.168.100.9:7007)**
|
||||
|
||||
สร้าง/แก้ไข `docker-compose.yml` ใน Airbyte directory:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
# ... existing Airbyte services ...
|
||||
|
||||
dozzle-agent:
|
||||
image: amir20/dozzle:latest
|
||||
container_name: dozzle-agent-airbyte
|
||||
command: agent
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
ports:
|
||||
- "7007:7007"
|
||||
environment:
|
||||
DOZZLE_LEVEL: info
|
||||
DOZZLE_HOSTNAME: Airbyte Server
|
||||
TZ: Asia/Bangkok
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- airbyte_network # ใช้ network ของ Airbyte
|
||||
```
|
||||
|
||||
**Start agent:**
|
||||
```bash
|
||||
docker compose up -d dozzle-agent
|
||||
```
|
||||
|
||||
#### **สำหรับ Airflow Host (192.168.100.9:7008)**
|
||||
|
||||
สร้าง/แก้ไข `docker-compose.yml` ใน Airflow directory:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
# ... existing Airflow services ...
|
||||
|
||||
dozzle-agent:
|
||||
image: amir20/dozzle:latest
|
||||
container_name: dozzle-agent-airflow
|
||||
command: agent
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
ports:
|
||||
- "7008:7007" # External: 7008, Internal: 7007
|
||||
environment:
|
||||
DOZZLE_LEVEL: info
|
||||
DOZZLE_HOSTNAME: Airflow Server
|
||||
TZ: Asia/Bangkok
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- shared_data_network # ใช้ network ของ Airflow
|
||||
```
|
||||
|
||||
**Start agent:**
|
||||
```bash
|
||||
docker compose up -d dozzle-agent
|
||||
```
|
||||
|
||||
### **Step 2: ตรวจสอบ Agents**
|
||||
|
||||
```bash
|
||||
# ตรวจสอบ Airbyte agent
|
||||
curl http://192.168.100.9:7007/healthcheck
|
||||
|
||||
# ตรวจสอบ Airflow agent
|
||||
curl http://192.168.100.9:7008/healthcheck
|
||||
|
||||
# ดู logs
|
||||
docker logs dozzle-agent-airbyte
|
||||
docker logs dozzle-agent-airflow
|
||||
```
|
||||
|
||||
### **Step 3: Start Dozzle Main UI (Main Server)**
|
||||
|
||||
```bash
|
||||
cd 01-infra
|
||||
docker compose up -d dozzle
|
||||
|
||||
# ตรวจสอบ
|
||||
docker logs dozzle -f
|
||||
```
|
||||
|
||||
### **Step 4: เข้าใช้งาน**
|
||||
|
||||
**Direct access:**
|
||||
```
|
||||
http://localhost:9999/dozzle
|
||||
```
|
||||
|
||||
**ผ่าน Nginx:**
|
||||
```
|
||||
http://ai.sriphat.com/dozzle
|
||||
```
|
||||
|
||||
## 🔧 Configuration Details
|
||||
|
||||
### **Main Server (.env.global)**
|
||||
|
||||
```bash
|
||||
# Dozzle - Docker Log Viewer & Monitoring
|
||||
DOZZLE_PORT=9999
|
||||
DOZZLE_LEVEL=info
|
||||
DOZZLE_BASE=/dozzle
|
||||
DOZZLE_HOSTNAME=Sriphat Main Server
|
||||
DOZZLE_AUTH_PROVIDER=none
|
||||
DOZZLE_RESTART_POLICY=unless-stopped
|
||||
|
||||
# Remote agents: Airbyte and Airflow on 192.168.100.9
|
||||
# Format: host:port,host:port (comma-separated)
|
||||
DOZZLE_REMOTE_AGENT=192.168.100.9:7007,192.168.100.9:7008
|
||||
```
|
||||
|
||||
### **Agent Configuration**
|
||||
|
||||
**Airbyte Agent:**
|
||||
- Port: 7007
|
||||
- Hostname: Airbyte Server
|
||||
- Monitors: Airbyte containers
|
||||
|
||||
**Airflow Agent:**
|
||||
- Port: 7008
|
||||
- Hostname: Airflow Server
|
||||
- Monitors: Airflow containers
|
||||
|
||||
## 🌐 Nginx Configuration
|
||||
|
||||
Dozzle config ถูกเพิ่มใน:
|
||||
- `01-infra/nginx-configs/dozzle.conf`
|
||||
- `01-infra/nginx-configs/complete-example.conf`
|
||||
|
||||
**ตั้งค่าใน Nginx Proxy Manager:**
|
||||
1. ไปที่ Proxy Host → Edit
|
||||
2. Tab "Advanced"
|
||||
3. เพิ่ม Dozzle config จาก `complete-example.conf`
|
||||
|
||||
## 🔍 Features
|
||||
|
||||
### **1. Multi-Host Monitoring**
|
||||
- ✅ ดู logs จาก Main Server
|
||||
- ✅ ดู logs จาก Airbyte Host (192.168.100.9:7007)
|
||||
- ✅ ดู logs จาก Airflow Host (192.168.100.9:7008)
|
||||
- ✅ Switch ระหว่าง hosts ผ่าน dropdown
|
||||
|
||||
### **2. Real-time Log Streaming**
|
||||
- Live log updates
|
||||
- Color-coded logs
|
||||
- JSON formatting
|
||||
- Multi-line grouping
|
||||
|
||||
### **3. Container Management**
|
||||
- View container stats (CPU, Memory, Network)
|
||||
- Start/Stop/Restart containers
|
||||
- Interactive shell access
|
||||
- Container filtering
|
||||
|
||||
### **4. Advanced Features**
|
||||
- Search และ filter logs
|
||||
- Download logs
|
||||
- Multiple container view
|
||||
- SQL-based log querying
|
||||
|
||||
## 🐛 Troubleshooting
|
||||
|
||||
### **Issue: Agent ไม่ปรากฏใน UI**
|
||||
|
||||
**ตรวจสอบ:**
|
||||
```bash
|
||||
# 1. Agent ทำงานหรือไม่
|
||||
docker ps | grep dozzle-agent
|
||||
|
||||
# 2. Port เปิดหรือไม่
|
||||
netstat -tulpn | grep 7007
|
||||
netstat -tulpn | grep 7008
|
||||
|
||||
# 3. Firewall
|
||||
sudo ufw status
|
||||
sudo ufw allow 7007
|
||||
sudo ufw allow 7008
|
||||
|
||||
# 4. Network connectivity
|
||||
ping 192.168.100.9
|
||||
telnet 192.168.100.9 7007
|
||||
telnet 192.168.100.9 7008
|
||||
```
|
||||
|
||||
### **Issue: Connection Refused**
|
||||
|
||||
**สาเหตุ:**
|
||||
- Agent ไม่ทำงาน
|
||||
- Firewall block port
|
||||
- Network ไม่เชื่อมต่อ
|
||||
|
||||
**วิธีแก้:**
|
||||
```bash
|
||||
# Restart agent
|
||||
docker restart dozzle-agent-airbyte
|
||||
docker restart dozzle-agent-airflow
|
||||
|
||||
# ตรวจสอบ logs
|
||||
docker logs dozzle-agent-airbyte
|
||||
docker logs dozzle-agent-airflow
|
||||
|
||||
# ทดสอบ connectivity
|
||||
curl http://192.168.100.9:7007/healthcheck
|
||||
curl http://192.168.100.9:7008/healthcheck
|
||||
```
|
||||
|
||||
### **Issue: Containers ไม่แสดงใน Agent**
|
||||
|
||||
**สาเหตุ:**
|
||||
- Docker socket ไม่ mount
|
||||
- Agent ไม่มี permission
|
||||
|
||||
**วิธีแก้:**
|
||||
```bash
|
||||
# ตรวจสอบ volume mount
|
||||
docker inspect dozzle-agent-airbyte | grep docker.sock
|
||||
|
||||
# ตรวจสอบ permissions
|
||||
ls -la /var/run/docker.sock
|
||||
|
||||
# Restart agent
|
||||
docker restart dozzle-agent-airbyte
|
||||
```
|
||||
|
||||
## 🔐 Security Considerations
|
||||
|
||||
### **1. Network Security**
|
||||
|
||||
**ใช้ Internal Network (แนะนำ):**
|
||||
```yaml
|
||||
# Agent ไม่ expose port ออกภายนอก
|
||||
# ใช้ Docker network แทน
|
||||
dozzle-agent:
|
||||
# ไม่ต้องมี ports section
|
||||
networks:
|
||||
- shared_data_network
|
||||
```
|
||||
|
||||
**Main UI เชื่อมต่อผ่าน network:**
|
||||
```yaml
|
||||
DOZZLE_REMOTE_AGENT=dozzle-agent-airbyte:7007,dozzle-agent-airflow:7007
|
||||
```
|
||||
|
||||
### **2. Firewall Rules**
|
||||
|
||||
```bash
|
||||
# อนุญาตเฉพาะ Main Server
|
||||
sudo ufw allow from <main-server-ip> to any port 7007
|
||||
sudo ufw allow from <main-server-ip> to any port 7008
|
||||
```
|
||||
|
||||
### **3. Authentication**
|
||||
|
||||
**Enable simple auth:**
|
||||
```yaml
|
||||
DOZZLE_AUTH_PROVIDER: simple
|
||||
```
|
||||
|
||||
สร้าง `01-infra/data/dozzle/users.yml`:
|
||||
```yaml
|
||||
users:
|
||||
- name: admin
|
||||
username: admin
|
||||
password: $2a$10$...
|
||||
email: admin@sriphat.com
|
||||
```
|
||||
|
||||
### **4. Read-only Docker Socket**
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
```
|
||||
|
||||
## 📊 Monitoring
|
||||
|
||||
### **Health Checks**
|
||||
|
||||
```bash
|
||||
# Main UI
|
||||
curl http://localhost:9999/dozzle/healthcheck
|
||||
|
||||
# Airbyte Agent
|
||||
curl http://192.168.100.9:7007/healthcheck
|
||||
|
||||
# Airflow Agent
|
||||
curl http://192.168.100.9:7008/healthcheck
|
||||
```
|
||||
|
||||
### **Logs**
|
||||
|
||||
```bash
|
||||
# Main UI
|
||||
docker logs dozzle -f
|
||||
|
||||
# Agents
|
||||
docker logs dozzle-agent-airbyte -f
|
||||
docker logs dozzle-agent-airflow -f
|
||||
```
|
||||
|
||||
## 🎯 Best Practices
|
||||
|
||||
1. **ใช้ Internal Network** - ไม่ expose agent ports ออกภายนอก
|
||||
2. **Enable Authentication** - ใช้ simple auth หรือ forward proxy
|
||||
3. **Monitor Agent Health** - ตั้ง healthcheck และ alerting
|
||||
4. **Backup Configuration** - backup `users.yml` และ `.env` files
|
||||
5. **Update Regularly** - อัพเดท Dozzle image เป็นประจำ
|
||||
6. **Use HTTPS** - ใช้ SSL/TLS สำหรับ production
|
||||
7. **Limit Access** - ใช้ firewall และ access lists
|
||||
|
||||
## 📚 References
|
||||
|
||||
- [Dozzle Documentation](https://dozzle.dev/)
|
||||
- [Agent Mode Guide](https://dozzle.dev/guide/agent)
|
||||
- [Authentication Guide](https://dozzle.dev/guide/authentication)
|
||||
- [Remote Hosts Guide](https://dozzle.dev/guide/remote-hosts)
|
||||
|
||||
## 🔄 Maintenance
|
||||
|
||||
### **Update Dozzle**
|
||||
|
||||
```bash
|
||||
# Main UI
|
||||
cd 01-infra
|
||||
docker compose pull dozzle
|
||||
docker compose up -d dozzle
|
||||
|
||||
# Agents
|
||||
docker pull amir20/dozzle:latest
|
||||
docker restart dozzle-agent-airbyte
|
||||
docker restart dozzle-agent-airflow
|
||||
```
|
||||
|
||||
### **Backup Configuration**
|
||||
|
||||
```bash
|
||||
# Backup .env
|
||||
cp .env.global .env.global.backup
|
||||
|
||||
# Backup users.yml (if using auth)
|
||||
cp 01-infra/data/dozzle/users.yml users.yml.backup
|
||||
```
|
||||
|
||||
## 🎉 Summary
|
||||
|
||||
**ตอนนี้คุณมี:**
|
||||
- ✅ Dozzle Main UI บน Main Server
|
||||
- ✅ Dozzle Agent บน Airbyte Host (192.168.100.9:7007)
|
||||
- ✅ Dozzle Agent บน Airflow Host (192.168.100.9:7008)
|
||||
- ✅ Nginx reverse proxy สำหรับ `/dozzle` subpath
|
||||
- ✅ Multi-host monitoring ผ่าน single UI
|
||||
- ✅ Real-time log streaming จากทุก hosts
|
||||
|
||||
**เข้าใช้งานที่:**
|
||||
```
|
||||
http://ai.sriphat.com/dozzle
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Monitor logs จาก Main Server, Airbyte, และ Airflow
|
||||
- Real-time streaming
|
||||
- Container stats
|
||||
- Interactive shell
|
||||
- Search และ filter
|
||||
Reference in New Issue
Block a user