feat: MinIO integration — bucket finance, API service upload, Nginx routing

- 01-infra/nginx-configs: add MinIO /minio/ and /minio-console/ location blocks
  (port 9000 S3 API, port 9001 Console UI, path stripping via rewrite)
- 03-apiservice: integrate MinIO minio-python SDK for file upload
  - requirements.txt: add minio==7.2.11
  - app/core/config.py: add MINIO_ENDPOINT, ACCESS_KEY, SECRET_KEY, BUCKET_FINANCE, USE_SSL
  - app/services/minio_client.py: new — upload_file(), get_presigned_url(), delete_file()
  - app/routes/pages.py: replace local /data/uploads/ write with MinIO upload to finance bucket
  - docker-compose.yml: pass MinIO env vars to container
  - .env.example: document MinIO vars
- 07-minio/.env.example: add MINIO_SVC_ACCESS_KEY/SECRET_KEY section
- 07-minio/README.md: add Python minio SDK and Airflow DAG usage guide
- CLAUDE.md: project context (servers, SSH, paths, service distribution)
- document-obsidiant/: initial Obsidian docs for all services
This commit is contained in:
jigoong
2026-05-20 17:42:39 +07:00
parent 9dcf24eeb7
commit a587be08bd
20 changed files with 2601 additions and 13 deletions

View File

@@ -47,3 +47,11 @@ KEYCLOAK_REDIRECT_URI=http://localhost:8040/apiservice/auth/callback
AIRFLOW_API_URL=http://airflow-webserver:8080
AIRFLOW_API_TOKEN=your-airflow-api-token
AIRFLOW_DAG_ID_FINANCE=process_finance_excel
# MinIO Object Storage (server 2: 192.168.100.9)
# ใช้ service account sp_service_ac (ไม่ใช้ root credentials)
MINIO_ENDPOINT=192.168.100.9:9000
MINIO_SVC_ACCESS_KEY=sp_service_ac
MINIO_SVC_SECRET_KEY=your-minio-service-account-secret
MINIO_BUCKET_FINANCE=finance
MINIO_USE_SSL=false

View File

@@ -48,5 +48,12 @@ class Settings(BaseSettings):
AIRFLOW_API_TOKEN: str = ""
AIRFLOW_DAG_ID_FINANCE: str = "process_finance_excel"
# MinIO Object Storage
MINIO_ENDPOINT: str = "192.168.100.9:9000"
MINIO_ACCESS_KEY: str = ""
MINIO_SECRET_KEY: str = ""
MINIO_BUCKET_FINANCE: str = "finance"
MINIO_USE_SSL: bool = False
settings = Settings()

View File

@@ -19,6 +19,7 @@ from app.security.permissions import require_role, Roles
from app.db.session import get_db
from app.models.upload import UploadHistory
from app.services.airflow_client import airflow_client
from app.services import minio_client
logger = logging.getLogger(__name__)
@@ -28,7 +29,7 @@ router = APIRouter()
templates_dir = Path(__file__).parent.parent / "templates"
templates = Jinja2Templates(directory=str(templates_dir))
# Upload directory
# Local fallback directory (used only if MinIO is not configured)
UPLOAD_DIR = Path("/data/uploads")
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
@@ -120,29 +121,37 @@ async def upload_finance_file(
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
safe_filename = file.filename.replace(" ", "_")
unique_filename = f"{timestamp}_{safe_filename}"
filepath = UPLOAD_DIR / unique_filename
# Save file
# Read file content
try:
content = await file.read()
with open(filepath, "wb") as f:
f.write(content)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to save file: {str(e)}"
raise HTTPException(status_code=500, detail=f"Failed to read file: {str(e)}")
# Upload to MinIO finance bucket
object_key = f"finance/{unique_filename}"
try:
minio_client.upload_file(
bucket=settings.MINIO_BUCKET_FINANCE,
object_name=object_key,
data=content,
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
)
filepath_stored = object_key # store MinIO key in DB
except Exception as e:
logger.error(f"MinIO upload failed: {e}")
raise HTTPException(status_code=500, detail=f"Failed to upload file to storage: {str(e)}")
# Get username from session
user = request.session.get("user")
username = user.get("username") if user else "anonymous"
# Create upload record in database
upload_id = f"upload_{timestamp}"
upload_record = UploadHistory(
upload_id=upload_id,
filename=file.filename,
filepath=str(filepath),
filepath=filepath_stored, # MinIO object key: finance/<filename>
description=description,
status="pending",
uploaded_by=username

View File

@@ -0,0 +1,48 @@
import io
import logging
from minio import Minio
from minio.error import S3Error
from app.core.config import settings
logger = logging.getLogger(__name__)
_client: Minio | None = None
def get_client() -> Minio:
global _client
if _client is None:
_client = Minio(
endpoint=settings.MINIO_ENDPOINT,
access_key=settings.MINIO_ACCESS_KEY,
secret_key=settings.MINIO_SECRET_KEY,
secure=settings.MINIO_USE_SSL,
)
return _client
def upload_file(bucket: str, object_name: str, data: bytes, content_type: str = "application/octet-stream") -> str:
"""Upload bytes to MinIO. Returns the object key."""
client = get_client()
client.put_object(
bucket_name=bucket,
object_name=object_name,
data=io.BytesIO(data),
length=len(data),
content_type=content_type,
)
logger.info(f"Uploaded {object_name} to bucket {bucket}")
return object_name
def get_presigned_url(bucket: str, object_name: str, expires_seconds: int = 3600) -> str:
"""Generate presigned GET URL valid for expires_seconds (default 1h)."""
from datetime import timedelta
client = get_client()
return client.presigned_get_object(bucket, object_name, expires=timedelta(seconds=expires_seconds))
def delete_file(bucket: str, object_name: str) -> None:
client = get_client()
client.remove_object(bucket, object_name)
logger.info(f"Deleted {object_name} from bucket {bucket}")

View File

@@ -30,6 +30,11 @@ services:
- KEYCLOAK_CLIENT_ID=${API_KEYCLOAK_CLIENT_ID}
- KEYCLOAK_CLIENT_SECRET=${API_KEYCLOAK_CLIENT_SECRET}
- KEYCLOAK_REDIRECT_URI=${API_KEYCLOAK_REDIRECT_URI}
- MINIO_ENDPOINT=${MINIO_ENDPOINT:-192.168.100.9:9000}
- MINIO_ACCESS_KEY=${MINIO_SVC_ACCESS_KEY}
- MINIO_SECRET_KEY=${MINIO_SVC_SECRET_KEY}
- MINIO_BUCKET_FINANCE=${MINIO_BUCKET_FINANCE:-finance}
- MINIO_USE_SSL=${MINIO_USE_SSL:-false}
- LOG_LEVEL=debug
ports:
- "8040:8040"

View File

@@ -17,4 +17,4 @@ cryptography==42.0.5
python-keycloak==3.9.0
Authlib==1.3.0
python-jose[cryptography]==3.3.0
minio==7.2.11