feat: MinIO integration — bucket finance, API service upload, Nginx routing
- 01-infra/nginx-configs: add MinIO /minio/ and /minio-console/ location blocks (port 9000 S3 API, port 9001 Console UI, path stripping via rewrite) - 03-apiservice: integrate MinIO minio-python SDK for file upload - requirements.txt: add minio==7.2.11 - app/core/config.py: add MINIO_ENDPOINT, ACCESS_KEY, SECRET_KEY, BUCKET_FINANCE, USE_SSL - app/services/minio_client.py: new — upload_file(), get_presigned_url(), delete_file() - app/routes/pages.py: replace local /data/uploads/ write with MinIO upload to finance bucket - docker-compose.yml: pass MinIO env vars to container - .env.example: document MinIO vars - 07-minio/.env.example: add MINIO_SVC_ACCESS_KEY/SECRET_KEY section - 07-minio/README.md: add Python minio SDK and Airflow DAG usage guide - CLAUDE.md: project context (servers, SSH, paths, service distribution) - document-obsidiant/: initial Obsidian docs for all services
This commit is contained in:
@@ -47,3 +47,11 @@ KEYCLOAK_REDIRECT_URI=http://localhost:8040/apiservice/auth/callback
|
||||
AIRFLOW_API_URL=http://airflow-webserver:8080
|
||||
AIRFLOW_API_TOKEN=your-airflow-api-token
|
||||
AIRFLOW_DAG_ID_FINANCE=process_finance_excel
|
||||
|
||||
# MinIO Object Storage (server 2: 192.168.100.9)
|
||||
# ใช้ service account sp_service_ac (ไม่ใช้ root credentials)
|
||||
MINIO_ENDPOINT=192.168.100.9:9000
|
||||
MINIO_SVC_ACCESS_KEY=sp_service_ac
|
||||
MINIO_SVC_SECRET_KEY=your-minio-service-account-secret
|
||||
MINIO_BUCKET_FINANCE=finance
|
||||
MINIO_USE_SSL=false
|
||||
|
||||
@@ -48,5 +48,12 @@ class Settings(BaseSettings):
|
||||
AIRFLOW_API_TOKEN: str = ""
|
||||
AIRFLOW_DAG_ID_FINANCE: str = "process_finance_excel"
|
||||
|
||||
# MinIO Object Storage
|
||||
MINIO_ENDPOINT: str = "192.168.100.9:9000"
|
||||
MINIO_ACCESS_KEY: str = ""
|
||||
MINIO_SECRET_KEY: str = ""
|
||||
MINIO_BUCKET_FINANCE: str = "finance"
|
||||
MINIO_USE_SSL: bool = False
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@@ -19,6 +19,7 @@ from app.security.permissions import require_role, Roles
|
||||
from app.db.session import get_db
|
||||
from app.models.upload import UploadHistory
|
||||
from app.services.airflow_client import airflow_client
|
||||
from app.services import minio_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,7 +29,7 @@ router = APIRouter()
|
||||
templates_dir = Path(__file__).parent.parent / "templates"
|
||||
templates = Jinja2Templates(directory=str(templates_dir))
|
||||
|
||||
# Upload directory
|
||||
# Local fallback directory (used only if MinIO is not configured)
|
||||
UPLOAD_DIR = Path("/data/uploads")
|
||||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -120,29 +121,37 @@ async def upload_finance_file(
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
safe_filename = file.filename.replace(" ", "_")
|
||||
unique_filename = f"{timestamp}_{safe_filename}"
|
||||
filepath = UPLOAD_DIR / unique_filename
|
||||
|
||||
# Save file
|
||||
|
||||
# Read file content
|
||||
try:
|
||||
content = await file.read()
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(content)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to save file: {str(e)}"
|
||||
raise HTTPException(status_code=500, detail=f"Failed to read file: {str(e)}")
|
||||
|
||||
# Upload to MinIO finance bucket
|
||||
object_key = f"finance/{unique_filename}"
|
||||
try:
|
||||
minio_client.upload_file(
|
||||
bucket=settings.MINIO_BUCKET_FINANCE,
|
||||
object_name=object_key,
|
||||
data=content,
|
||||
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
)
|
||||
|
||||
filepath_stored = object_key # store MinIO key in DB
|
||||
except Exception as e:
|
||||
logger.error(f"MinIO upload failed: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Failed to upload file to storage: {str(e)}")
|
||||
|
||||
# Get username from session
|
||||
user = request.session.get("user")
|
||||
username = user.get("username") if user else "anonymous"
|
||||
|
||||
|
||||
# Create upload record in database
|
||||
upload_id = f"upload_{timestamp}"
|
||||
upload_record = UploadHistory(
|
||||
upload_id=upload_id,
|
||||
filename=file.filename,
|
||||
filepath=str(filepath),
|
||||
filepath=filepath_stored, # MinIO object key: finance/<filename>
|
||||
description=description,
|
||||
status="pending",
|
||||
uploaded_by=username
|
||||
|
||||
48
03-apiservice/app/services/minio_client.py
Normal file
48
03-apiservice/app/services/minio_client.py
Normal file
@@ -0,0 +1,48 @@
|
||||
import io
|
||||
import logging
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_client: Minio | None = None
|
||||
|
||||
|
||||
def get_client() -> Minio:
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = Minio(
|
||||
endpoint=settings.MINIO_ENDPOINT,
|
||||
access_key=settings.MINIO_ACCESS_KEY,
|
||||
secret_key=settings.MINIO_SECRET_KEY,
|
||||
secure=settings.MINIO_USE_SSL,
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
def upload_file(bucket: str, object_name: str, data: bytes, content_type: str = "application/octet-stream") -> str:
|
||||
"""Upload bytes to MinIO. Returns the object key."""
|
||||
client = get_client()
|
||||
client.put_object(
|
||||
bucket_name=bucket,
|
||||
object_name=object_name,
|
||||
data=io.BytesIO(data),
|
||||
length=len(data),
|
||||
content_type=content_type,
|
||||
)
|
||||
logger.info(f"Uploaded {object_name} to bucket {bucket}")
|
||||
return object_name
|
||||
|
||||
|
||||
def get_presigned_url(bucket: str, object_name: str, expires_seconds: int = 3600) -> str:
|
||||
"""Generate presigned GET URL valid for expires_seconds (default 1h)."""
|
||||
from datetime import timedelta
|
||||
client = get_client()
|
||||
return client.presigned_get_object(bucket, object_name, expires=timedelta(seconds=expires_seconds))
|
||||
|
||||
|
||||
def delete_file(bucket: str, object_name: str) -> None:
|
||||
client = get_client()
|
||||
client.remove_object(bucket, object_name)
|
||||
logger.info(f"Deleted {object_name} from bucket {bucket}")
|
||||
@@ -30,6 +30,11 @@ services:
|
||||
- KEYCLOAK_CLIENT_ID=${API_KEYCLOAK_CLIENT_ID}
|
||||
- KEYCLOAK_CLIENT_SECRET=${API_KEYCLOAK_CLIENT_SECRET}
|
||||
- KEYCLOAK_REDIRECT_URI=${API_KEYCLOAK_REDIRECT_URI}
|
||||
- MINIO_ENDPOINT=${MINIO_ENDPOINT:-192.168.100.9:9000}
|
||||
- MINIO_ACCESS_KEY=${MINIO_SVC_ACCESS_KEY}
|
||||
- MINIO_SECRET_KEY=${MINIO_SVC_SECRET_KEY}
|
||||
- MINIO_BUCKET_FINANCE=${MINIO_BUCKET_FINANCE:-finance}
|
||||
- MINIO_USE_SSL=${MINIO_USE_SSL:-false}
|
||||
- LOG_LEVEL=debug
|
||||
ports:
|
||||
- "8040:8040"
|
||||
|
||||
@@ -17,4 +17,4 @@ cryptography==42.0.5
|
||||
python-keycloak==3.9.0
|
||||
Authlib==1.3.0
|
||||
python-jose[cryptography]==3.3.0
|
||||
|
||||
minio==7.2.11
|
||||
|
||||
Reference in New Issue
Block a user