feat: MinIO integration — bucket finance, API service upload, Nginx routing
- 01-infra/nginx-configs: add MinIO /minio/ and /minio-console/ location blocks (port 9000 S3 API, port 9001 Console UI, path stripping via rewrite) - 03-apiservice: integrate MinIO minio-python SDK for file upload - requirements.txt: add minio==7.2.11 - app/core/config.py: add MINIO_ENDPOINT, ACCESS_KEY, SECRET_KEY, BUCKET_FINANCE, USE_SSL - app/services/minio_client.py: new — upload_file(), get_presigned_url(), delete_file() - app/routes/pages.py: replace local /data/uploads/ write with MinIO upload to finance bucket - docker-compose.yml: pass MinIO env vars to container - .env.example: document MinIO vars - 07-minio/.env.example: add MINIO_SVC_ACCESS_KEY/SECRET_KEY section - 07-minio/README.md: add Python minio SDK and Airflow DAG usage guide - CLAUDE.md: project context (servers, SSH, paths, service distribution) - document-obsidiant/: initial Obsidian docs for all services
This commit is contained in:
@@ -49,6 +49,14 @@ MINIO_IDENTITY_OPENID_SCOPES=openid,profile,email
|
||||
# Redirect URI after authentication
|
||||
MINIO_IDENTITY_OPENID_REDIRECT_URI=https://ai.sriphat.com/minio-console/oauth_callback
|
||||
|
||||
# ============================================================================
|
||||
# Service Account — Web Service
|
||||
# สร้างหลัง MinIO start แล้วด้วย mc CLI
|
||||
# mc admin user svcacct add --access-key <KEY> --secret-key <SECRET> sriphat admin
|
||||
# ============================================================================
|
||||
MINIO_SVC_ACCESS_KEY=sp_service_ac
|
||||
MINIO_SVC_SECRET_KEY=your-service-account-secret-here
|
||||
|
||||
# ============================================================================
|
||||
# Timezone
|
||||
# ============================================================================
|
||||
|
||||
@@ -166,6 +166,133 @@ mc ls myminio/my-bucket
|
||||
mc rm myminio/my-bucket/myfile.txt
|
||||
```
|
||||
|
||||
### **Python SDK (minio — แนะนำสำหรับ Sriphat Platform)**
|
||||
|
||||
ใช้ `minio` package (Official MinIO Python SDK) แทน boto3 สำหรับ internal services:
|
||||
|
||||
```python
|
||||
from minio import Minio
|
||||
import io
|
||||
|
||||
# Connection — ใช้ internal IP จาก service บน server อื่น
|
||||
client = Minio(
|
||||
endpoint="192.168.100.9:9000", # internal IP, ไม่ใช่ public URL
|
||||
access_key="sp_service_ac",
|
||||
secret_key="<MINIO_SVC_SECRET_KEY>",
|
||||
secure=False, # HTTP ภายใน network
|
||||
)
|
||||
|
||||
# Upload file
|
||||
with open("report.xlsx", "rb") as f:
|
||||
data = f.read()
|
||||
client.put_object(
|
||||
bucket_name="finance",
|
||||
object_name="finance/20260520_report.xlsx",
|
||||
data=io.BytesIO(data),
|
||||
length=len(data),
|
||||
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
)
|
||||
|
||||
# Download file (คืนค่าเป็น HTTPResponse)
|
||||
response = client.get_object("finance", "finance/20260520_report.xlsx")
|
||||
data = response.read()
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
# Download ไปยัง file โดยตรง
|
||||
client.fget_object("finance", "finance/20260520_report.xlsx", "/tmp/report.xlsx")
|
||||
|
||||
# List objects ใน bucket
|
||||
for obj in client.list_objects("finance", prefix="finance/", recursive=True):
|
||||
print(obj.object_name, obj.size)
|
||||
|
||||
# Generate presigned URL (สำหรับให้ภายนอกดาวน์โหลด ใช้ได้ 1 ชั่วโมง)
|
||||
from datetime import timedelta
|
||||
url = client.presigned_get_object("finance", "finance/report.xlsx", expires=timedelta(hours=1))
|
||||
print(url)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### **Airflow DAG — อ่านไฟล์จาก MinIO finance bucket**
|
||||
|
||||
Airflow อยู่บน server .9 (server เดียวกับ MinIO) ใช้ container name `minio:9000` หรือ `192.168.100.9:9000`:
|
||||
|
||||
```python
|
||||
from minio import Minio
|
||||
import pandas as pd
|
||||
import io
|
||||
from airflow.decorators import dag, task
|
||||
from airflow.utils.dates import days_ago
|
||||
|
||||
@dag(schedule=None, start_date=days_ago(1), catchup=False)
|
||||
def process_finance_excel():
|
||||
|
||||
@task
|
||||
def download_and_process(filepath: str, **context):
|
||||
"""
|
||||
filepath = MinIO object key เช่น "finance/20260520_123000_report.xlsx"
|
||||
ส่งมาจาก API Service ผ่าน DAG trigger conf
|
||||
"""
|
||||
client = Minio(
|
||||
endpoint="minio:9000", # container name บน shared_data_network
|
||||
access_key="sp_service_ac", # ใช้ service account เดียวกับ API
|
||||
secret_key="{{ var.value.MINIO_SVC_SECRET_KEY }}", # เก็บใน Airflow Variables
|
||||
secure=False,
|
||||
)
|
||||
|
||||
# Download file จาก MinIO
|
||||
response = client.get_object(bucket_name="finance", object_name=filepath)
|
||||
file_bytes = response.read()
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
# ประมวลผลด้วย pandas
|
||||
df = pd.read_excel(io.BytesIO(file_bytes))
|
||||
print(f"Loaded {len(df)} rows from {filepath}")
|
||||
|
||||
# ... process data ...
|
||||
return {"rows": len(df), "filepath": filepath}
|
||||
|
||||
@task
|
||||
def get_filepath(**context):
|
||||
conf = context["dag_run"].conf or {}
|
||||
return conf.get("filepath", "")
|
||||
|
||||
fp = get_filepath()
|
||||
download_and_process(fp)
|
||||
|
||||
process_finance_excel()
|
||||
```
|
||||
|
||||
**ตั้งค่า Airflow Connection (ทางเลือก — ใช้ S3Hook)**
|
||||
|
||||
ถ้าต้องการใช้ `S3Hook` หรือ Airflow Operators:
|
||||
```
|
||||
Connection ID : minio_s3
|
||||
Connection Type: Amazon Web Services
|
||||
Extra (JSON) : {"endpoint_url": "http://minio:9000", "region_name": "ap-southeast-1"}
|
||||
Login : sp_service_ac
|
||||
Password : <MINIO_SVC_SECRET_KEY>
|
||||
```
|
||||
|
||||
```python
|
||||
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
|
||||
|
||||
hook = S3Hook(aws_conn_id="minio_s3")
|
||||
obj = hook.get_key(key="finance/20260520_report.xlsx", bucket_name="finance")
|
||||
data = obj.get()["Body"].read()
|
||||
df = pd.read_excel(io.BytesIO(data))
|
||||
```
|
||||
|
||||
**Airflow Variables ที่ต้องสร้าง:**
|
||||
| Key | Value |
|
||||
|-----|-------|
|
||||
| `MINIO_ENDPOINT` | `192.168.100.9:9000` |
|
||||
| `MINIO_SVC_SECRET_KEY` | (ดูจาก `07-minio/.env`) |
|
||||
|
||||
---
|
||||
|
||||
### **Python SDK (boto3)**
|
||||
|
||||
```python
|
||||
|
||||
Reference in New Issue
Block a user