update configuration docker setup for data platform
This commit is contained in:
jigoong
55
07-minio/.env.example
Normal file
55
07-minio/.env.example
Normal file
@@ -0,0 +1,55 @@
|
||||
# MinIO Configuration
|
||||
# Copy this file to .env and update with your values
|
||||
|
||||
# ============================================================================
|
||||
# MinIO Credentials
|
||||
# ============================================================================
|
||||
MINIO_ROOT_USER=minioadmin
|
||||
MINIO_ROOT_PASSWORD=minioadmin_secure_password_2026
|
||||
|
||||
# ============================================================================
|
||||
# MinIO Ports
|
||||
# ============================================================================
|
||||
MINIO_API_PORT=9000
|
||||
MINIO_CONSOLE_PORT=9001
|
||||
|
||||
# ============================================================================
|
||||
# MinIO Server URLs (for reverse proxy)
|
||||
# ============================================================================
|
||||
# API endpoint URL (for S3 API access)
|
||||
MINIO_SERVER_URL=https://ai.sriphat.com/minio
|
||||
|
||||
# Console UI URL (for web interface)
|
||||
MINIO_BROWSER_REDIRECT_URL=https://ai.sriphat.com/minio-console
|
||||
|
||||
# ============================================================================
|
||||
# MinIO Region
|
||||
# ============================================================================
|
||||
MINIO_REGION=ap-southeast-1
|
||||
|
||||
# ============================================================================
|
||||
# Keycloak Integration (OpenID Connect)
|
||||
# ============================================================================
|
||||
# Keycloak OpenID configuration URL
|
||||
# Format: https://{keycloak-domain}/realms/{realm-name}/.well-known/openid-configuration
|
||||
MINIO_IDENTITY_OPENID_CONFIG_URL=https://ai.sriphat.com/keycloak/realms/sriphat/.well-known/openid-configuration
|
||||
|
||||
# MinIO client in Keycloak
|
||||
MINIO_IDENTITY_OPENID_CLIENT_ID=minio
|
||||
|
||||
# Client secret from Keycloak
|
||||
MINIO_IDENTITY_OPENID_CLIENT_SECRET=your-minio-client-secret-here
|
||||
|
||||
# Claim name for policy mapping (default: policy)
|
||||
MINIO_IDENTITY_OPENID_CLAIM_NAME=policy
|
||||
|
||||
# OpenID scopes
|
||||
MINIO_IDENTITY_OPENID_SCOPES=openid,profile,email
|
||||
|
||||
# Redirect URI after authentication
|
||||
MINIO_IDENTITY_OPENID_REDIRECT_URI=https://ai.sriphat.com/minio-console/oauth_callback
|
||||
|
||||
# ============================================================================
|
||||
# Timezone
|
||||
# ============================================================================
|
||||
TZ=Asia/Bangkok
|
||||
30
07-minio/.gitignore
vendored
Normal file
30
07-minio/.gitignore
vendored
Normal file
@@ -0,0 +1,30 @@
|
||||
# Environment variables
|
||||
.env
|
||||
|
||||
# Data directory (persistent storage)
|
||||
data/
|
||||
|
||||
# SSL certificates
|
||||
certs/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Backup files
|
||||
*.tar.gz
|
||||
*.zip
|
||||
|
||||
# Temporary files
|
||||
*.tmp
|
||||
*.temp
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# IDE files
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
362
07-minio/KEYCLOAK_INTEGRATION.md
Normal file
362
07-minio/KEYCLOAK_INTEGRATION.md
Normal file
@@ -0,0 +1,362 @@
|
||||
# MinIO Keycloak Integration Guide
|
||||
|
||||
Complete guide for integrating MinIO with Keycloak for SSO authentication.
|
||||
|
||||
## 🎯 Overview
|
||||
|
||||
MinIO supports OpenID Connect (OIDC) authentication, allowing users to log in to MinIO Console using Keycloak credentials. This integration provides:
|
||||
|
||||
- **Single Sign-On (SSO)** - Users authenticate once with Keycloak
|
||||
- **Centralized User Management** - Manage users in Keycloak
|
||||
- **Role-Based Access Control** - Map Keycloak roles to MinIO policies
|
||||
- **Secure Authentication** - OAuth 2.0 / OpenID Connect flow
|
||||
|
||||
## 📋 Prerequisites
|
||||
|
||||
- Keycloak instance running and accessible
|
||||
- MinIO instance running
|
||||
- Admin access to both Keycloak and MinIO
|
||||
|
||||
## 🔧 Setup Steps
|
||||
|
||||
### **Step 1: Create MinIO Client in Keycloak**
|
||||
|
||||
1. **Login to Keycloak Admin Console**
|
||||
```
|
||||
https://ai.sriphat.com/keycloak
|
||||
```
|
||||
|
||||
2. **Select Realm**
|
||||
- Go to your realm (e.g., `sriphat`)
|
||||
|
||||
3. **Create Client**
|
||||
- Navigate to: **Clients** → **Create Client**
|
||||
- **Client ID**: `minio`
|
||||
- **Client Type**: `OpenID Connect`
|
||||
- **Client Protocol**: `openid-connect`
|
||||
- Click **Next**
|
||||
|
||||
4. **Capability Config**
|
||||
- **Client authentication**: `ON`
|
||||
- **Authorization**: `OFF`
|
||||
- **Authentication flow**:
|
||||
- ✅ Standard flow
|
||||
- ✅ Direct access grants
|
||||
- ❌ Implicit flow
|
||||
- ❌ Service accounts roles
|
||||
- Click **Next**
|
||||
|
||||
5. **Login Settings**
|
||||
- **Root URL**: `https://ai.sriphat.com/minio-console`
|
||||
- **Home URL**: `https://ai.sriphat.com/minio-console`
|
||||
- **Valid redirect URIs**:
|
||||
```
|
||||
https://ai.sriphat.com/minio-console/*
|
||||
https://ai.sriphat.com/minio-console/oauth_callback
|
||||
```
|
||||
- **Valid post logout redirect URIs**: `https://ai.sriphat.com/minio-console`
|
||||
- **Web origins**: `https://ai.sriphat.com`
|
||||
- Click **Save**
|
||||
|
||||
6. **Get Client Secret**
|
||||
- Go to **Credentials** tab
|
||||
- Copy the **Client Secret**
|
||||
- Save this for `.env` configuration
|
||||
|
||||
### **Step 2: Create Client Scope for MinIO Policy**
|
||||
|
||||
1. **Create Client Scope**
|
||||
- Navigate to: **Client Scopes** → **Create client scope**
|
||||
- **Name**: `minio-authorization`
|
||||
- **Type**: `Optional`
|
||||
- **Protocol**: `OpenID Connect`
|
||||
- **Display on consent screen**: `OFF`
|
||||
- Click **Save**
|
||||
|
||||
2. **Add Mapper for Policy Claim**
|
||||
- Go to **Mappers** tab
|
||||
- Click **Add mapper** → **By configuration**
|
||||
- Select **User Attribute**
|
||||
- **Name**: `minio-policy`
|
||||
- **User Attribute**: `minio_policy`
|
||||
- **Token Claim Name**: `policy`
|
||||
- **Claim JSON Type**: `String`
|
||||
- **Add to ID token**: `ON`
|
||||
- **Add to access token**: `ON`
|
||||
- **Add to userinfo**: `ON`
|
||||
- Click **Save**
|
||||
|
||||
3. **Assign Scope to MinIO Client**
|
||||
- Go to **Clients** → `minio`
|
||||
- Go to **Client scopes** tab
|
||||
- Click **Add client scope**
|
||||
- Select `minio-authorization`
|
||||
- Choose **Optional**
|
||||
- Click **Add**
|
||||
|
||||
### **Step 3: Create MinIO Policies in Keycloak**
|
||||
|
||||
MinIO uses policies to control access. Common policies:
|
||||
|
||||
- `consoleAdmin` - Full admin access
|
||||
- `readonly` - Read-only access
|
||||
- `readwrite` - Read and write access
|
||||
- `diagnostics` - Diagnostics access
|
||||
|
||||
**Add Policy to Users:**
|
||||
|
||||
1. **Go to Users**
|
||||
- Navigate to: **Users** → Select user
|
||||
|
||||
2. **Add Attribute**
|
||||
- Go to **Attributes** tab
|
||||
- Click **Add attribute**
|
||||
- **Key**: `minio_policy`
|
||||
- **Value**: `consoleAdmin` (or other policy)
|
||||
- Click **Save**
|
||||
|
||||
### **Step 4: Configure MinIO Environment Variables**
|
||||
|
||||
Update `07-minio/.env`:
|
||||
|
||||
```bash
|
||||
# Keycloak Integration
|
||||
MINIO_IDENTITY_OPENID_CONFIG_URL=https://ai.sriphat.com/keycloak/realms/sriphat/.well-known/openid-configuration
|
||||
MINIO_IDENTITY_OPENID_CLIENT_ID=minio
|
||||
MINIO_IDENTITY_OPENID_CLIENT_SECRET=your-client-secret-from-step-1
|
||||
MINIO_IDENTITY_OPENID_CLAIM_NAME=policy
|
||||
MINIO_IDENTITY_OPENID_SCOPES=openid,profile,email,minio-authorization
|
||||
MINIO_IDENTITY_OPENID_REDIRECT_URI=https://ai.sriphat.com/minio-console/oauth_callback
|
||||
```
|
||||
|
||||
### **Step 5: Restart MinIO**
|
||||
|
||||
```bash
|
||||
cd 07-minio
|
||||
docker compose down
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### **Step 6: Test Authentication**
|
||||
|
||||
1. **Access MinIO Console**
|
||||
```
|
||||
https://ai.sriphat.com/minio-console
|
||||
```
|
||||
|
||||
2. **Click "Login with SSO"**
|
||||
- You'll be redirected to Keycloak
|
||||
- Login with Keycloak credentials
|
||||
- After successful authentication, you'll be redirected back to MinIO Console
|
||||
|
||||
## 🔐 MinIO Policies
|
||||
|
||||
### **Default Policies**
|
||||
|
||||
MinIO comes with built-in policies:
|
||||
|
||||
| Policy | Description |
|
||||
|--------|-------------|
|
||||
| `consoleAdmin` | Full admin access to console and buckets |
|
||||
| `readonly` | Read-only access to buckets |
|
||||
| `readwrite` | Read and write access to buckets |
|
||||
| `diagnostics` | Access to diagnostics and monitoring |
|
||||
| `writeonly` | Write-only access (upload only) |
|
||||
|
||||
### **Custom Policies**
|
||||
|
||||
Create custom policies in MinIO Console or via `mc` CLI:
|
||||
|
||||
```bash
|
||||
# Install mc (MinIO Client)
|
||||
wget https://dl.min.io/client/mc/release/linux-amd64/mc
|
||||
chmod +x mc
|
||||
sudo mv mc /usr/local/bin/
|
||||
|
||||
# Configure mc
|
||||
mc alias set myminio https://ai.sriphat.com/minio minioadmin minioadmin_secure_password_2026
|
||||
|
||||
# Create custom policy
|
||||
cat > custom-policy.json <<EOF
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::mybucket/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# Add policy to MinIO
|
||||
mc admin policy create myminio custom-policy custom-policy.json
|
||||
```
|
||||
|
||||
## 🔄 Policy Mapping Flow
|
||||
|
||||
```
|
||||
User logs in with Keycloak
|
||||
↓
|
||||
Keycloak returns ID token with 'policy' claim
|
||||
↓
|
||||
MinIO reads 'policy' claim value (e.g., "consoleAdmin")
|
||||
↓
|
||||
MinIO applies corresponding policy to user session
|
||||
↓
|
||||
User has permissions defined by the policy
|
||||
```
|
||||
|
||||
## 🎯 Role-Based Access Example
|
||||
|
||||
### **Scenario: Different User Roles**
|
||||
|
||||
**Admin Users:**
|
||||
```
|
||||
Keycloak User Attribute:
|
||||
minio_policy: consoleAdmin
|
||||
```
|
||||
|
||||
**Data Scientists:**
|
||||
```
|
||||
Keycloak User Attribute:
|
||||
minio_policy: readwrite
|
||||
```
|
||||
|
||||
**Analysts:**
|
||||
```
|
||||
Keycloak User Attribute:
|
||||
minio_policy: readonly
|
||||
```
|
||||
|
||||
## 🐛 Troubleshooting
|
||||
|
||||
### **Issue: "Login with SSO" button not showing**
|
||||
|
||||
**Check:**
|
||||
```bash
|
||||
# Verify environment variables
|
||||
docker exec minio printenv | grep MINIO_IDENTITY_OPENID
|
||||
|
||||
# Check MinIO logs
|
||||
docker logs minio
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Ensure all `MINIO_IDENTITY_OPENID_*` variables are set
|
||||
- Restart MinIO container
|
||||
|
||||
### **Issue: Redirect loop after login**
|
||||
|
||||
**Check:**
|
||||
- `MINIO_BROWSER_REDIRECT_URL` matches Keycloak redirect URI
|
||||
- Valid redirect URIs in Keycloak client include `/oauth_callback`
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Update .env
|
||||
MINIO_BROWSER_REDIRECT_URL=https://ai.sriphat.com/minio-console
|
||||
MINIO_IDENTITY_OPENID_REDIRECT_URI=https://ai.sriphat.com/minio-console/oauth_callback
|
||||
```
|
||||
|
||||
### **Issue: User has no permissions after login**
|
||||
|
||||
**Check:**
|
||||
- User has `minio_policy` attribute in Keycloak
|
||||
- Policy name matches MinIO policy exactly (case-sensitive)
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Verify user attribute in Keycloak
|
||||
# Add minio_policy attribute with value: consoleAdmin
|
||||
```
|
||||
|
||||
### **Issue: Cannot access Keycloak config URL**
|
||||
|
||||
**Check:**
|
||||
```bash
|
||||
# Test from MinIO container
|
||||
docker exec minio curl -k https://ai.sriphat.com/keycloak/realms/sriphat/.well-known/openid-configuration
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Ensure MinIO container can reach Keycloak
|
||||
- Check network connectivity
|
||||
- Verify Keycloak realm name is correct
|
||||
|
||||
## 📊 Monitoring
|
||||
|
||||
### **Check OpenID Configuration**
|
||||
|
||||
```bash
|
||||
# View current OpenID config
|
||||
docker exec minio mc admin config get myminio identity_openid
|
||||
```
|
||||
|
||||
### **View Active Sessions**
|
||||
|
||||
```bash
|
||||
# List active user sessions
|
||||
docker exec minio mc admin user list myminio
|
||||
```
|
||||
|
||||
### **Audit Logs**
|
||||
|
||||
```bash
|
||||
# Enable audit logging
|
||||
docker exec minio mc admin config set myminio audit_webhook:1 endpoint="http://your-webhook-endpoint"
|
||||
|
||||
# View logs
|
||||
docker logs minio -f
|
||||
```
|
||||
|
||||
## 🔒 Security Best Practices
|
||||
|
||||
1. **Use HTTPS Only**
|
||||
- Always use HTTPS for MinIO and Keycloak
|
||||
- Configure SSL certificates properly
|
||||
|
||||
2. **Rotate Client Secrets**
|
||||
- Periodically rotate Keycloak client secrets
|
||||
- Update MinIO configuration after rotation
|
||||
|
||||
3. **Least Privilege Principle**
|
||||
- Assign minimal required policies to users
|
||||
- Use custom policies for specific use cases
|
||||
|
||||
4. **Monitor Access**
|
||||
- Enable audit logging
|
||||
- Review access logs regularly
|
||||
|
||||
5. **Secure Network**
|
||||
- Use firewall rules to restrict access
|
||||
- Consider VPN for sensitive data
|
||||
|
||||
## 📚 References
|
||||
|
||||
- [MinIO OpenID Connect](https://min.io/docs/minio/linux/operations/external-iam/configure-openid-external-identity-management.html)
|
||||
- [Keycloak OpenID Connect](https://www.keycloak.org/docs/latest/server_admin/#_oidc)
|
||||
- [MinIO IAM Policies](https://min.io/docs/minio/linux/administration/identity-access-management/policy-based-access-control.html)
|
||||
|
||||
## 🎉 Summary
|
||||
|
||||
After completing these steps:
|
||||
- ✅ MinIO integrated with Keycloak SSO
|
||||
- ✅ Users can login with Keycloak credentials
|
||||
- ✅ Role-based access control configured
|
||||
- ✅ Centralized user management in Keycloak
|
||||
- ✅ Secure HTTPS access via Nginx reverse proxy
|
||||
|
||||
**Access MinIO Console:**
|
||||
```
|
||||
https://ai.sriphat.com/minio-console
|
||||
```
|
||||
|
||||
**Login with SSO** → Keycloak authentication → MinIO Console access! 🚀
|
||||
520
07-minio/README.md
Normal file
520
07-minio/README.md
Normal file
@@ -0,0 +1,520 @@
|
||||
# MinIO Object Storage Service
|
||||
|
||||
MinIO is a high-performance, S3-compatible object storage system. This setup includes persistent storage, HTTPS access via Nginx reverse proxy, and Keycloak SSO integration.
|
||||
|
||||
## 🎯 Overview
|
||||
|
||||
**MinIO Features:**
|
||||
- **S3-Compatible API** - Works with AWS S3 SDKs and tools
|
||||
- **High Performance** - Optimized for large-scale data workloads
|
||||
- **Distributed Storage** - Supports multi-node deployment
|
||||
- **Web Console** - User-friendly web interface
|
||||
- **Encryption** - Server-side and client-side encryption
|
||||
- **Versioning** - Object versioning support
|
||||
- **Lifecycle Management** - Automatic data retention policies
|
||||
|
||||
**This Setup Includes:**
|
||||
- Docker Compose configuration
|
||||
- Persistent storage with volume mounts
|
||||
- HTTPS access via Nginx reverse proxy
|
||||
- Keycloak SSO integration (OpenID Connect)
|
||||
- Health checks and monitoring
|
||||
|
||||
## 📋 Prerequisites
|
||||
|
||||
- Docker and Docker Compose installed
|
||||
- Network: `shared_data_network` created
|
||||
- Nginx reverse proxy configured
|
||||
- Keycloak instance running (for SSO)
|
||||
- Server: 192.168.100.9
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### **Step 1: Configure Environment**
|
||||
|
||||
```bash
|
||||
cd 07-minio
|
||||
|
||||
# Copy example environment file
|
||||
cp .env.example .env
|
||||
|
||||
# Edit .env with your settings
|
||||
nano .env
|
||||
```
|
||||
|
||||
**Required Configuration:**
|
||||
```bash
|
||||
# MinIO Credentials
|
||||
MINIO_ROOT_USER=minioadmin
|
||||
MINIO_ROOT_PASSWORD=your-secure-password-here
|
||||
|
||||
# Keycloak Integration
|
||||
MINIO_IDENTITY_OPENID_CLIENT_SECRET=your-keycloak-client-secret
|
||||
```
|
||||
|
||||
### **Step 2: Create Data Directory**
|
||||
|
||||
```bash
|
||||
# Create persistent storage directory
|
||||
mkdir -p data
|
||||
|
||||
# Set permissions
|
||||
chmod 755 data
|
||||
```
|
||||
|
||||
### **Step 3: Start MinIO**
|
||||
|
||||
```bash
|
||||
# Start service
|
||||
docker compose up -d
|
||||
|
||||
# Check status
|
||||
docker compose ps
|
||||
|
||||
# View logs
|
||||
docker logs minio -f
|
||||
```
|
||||
|
||||
### **Step 4: Configure Nginx Reverse Proxy**
|
||||
|
||||
Add the configuration from `nginx-minio.conf` to your Nginx Proxy Manager:
|
||||
|
||||
1. Go to Nginx Proxy Manager UI
|
||||
2. Create/Edit Proxy Host for `ai.sriphat.com`
|
||||
3. Add MinIO configuration to "Custom Nginx Configuration"
|
||||
4. Save and test
|
||||
|
||||
### **Step 5: Setup Keycloak Integration**
|
||||
|
||||
Follow the detailed guide in `KEYCLOAK_INTEGRATION.md`:
|
||||
|
||||
1. Create MinIO client in Keycloak
|
||||
2. Configure client scopes and mappers
|
||||
3. Add policy attributes to users
|
||||
4. Update MinIO environment variables
|
||||
5. Restart MinIO service
|
||||
|
||||
## 🌐 Access URLs
|
||||
|
||||
**MinIO Console (Web UI):**
|
||||
```
|
||||
https://ai.sriphat.com/minio-console
|
||||
```
|
||||
|
||||
**MinIO API (S3 Compatible):**
|
||||
```
|
||||
https://ai.sriphat.com/minio
|
||||
```
|
||||
|
||||
**Direct Access (Development):**
|
||||
```
|
||||
http://192.168.100.9:9001 (Console)
|
||||
http://192.168.100.9:9000 (API)
|
||||
```
|
||||
|
||||
## 🔑 Authentication
|
||||
|
||||
### **Option 1: Root Credentials (Default)**
|
||||
|
||||
Login with root credentials from `.env`:
|
||||
- **Username**: Value of `MINIO_ROOT_USER`
|
||||
- **Password**: Value of `MINIO_ROOT_PASSWORD`
|
||||
|
||||
### **Option 2: Keycloak SSO (Recommended)**
|
||||
|
||||
1. Click "Login with SSO" on MinIO Console
|
||||
2. Authenticate with Keycloak
|
||||
3. Access granted based on policy mapping
|
||||
|
||||
See `KEYCLOAK_INTEGRATION.md` for setup instructions.
|
||||
|
||||
## 📦 Using MinIO
|
||||
|
||||
### **Web Console**
|
||||
|
||||
1. Access: `https://ai.sriphat.com/minio-console`
|
||||
2. Login with credentials or SSO
|
||||
3. Create buckets, upload files, manage access
|
||||
|
||||
### **MinIO Client (mc)**
|
||||
|
||||
```bash
|
||||
# Install mc
|
||||
wget https://dl.min.io/client/mc/release/linux-amd64/mc
|
||||
chmod +x mc
|
||||
sudo mv mc /usr/local/bin/
|
||||
|
||||
# Configure alias
|
||||
mc alias set myminio https://ai.sriphat.com/minio minioadmin your-password
|
||||
|
||||
# List buckets
|
||||
mc ls myminio
|
||||
|
||||
# Create bucket
|
||||
mc mb myminio/my-bucket
|
||||
|
||||
# Upload file
|
||||
mc cp myfile.txt myminio/my-bucket/
|
||||
|
||||
# Download file
|
||||
mc cp myminio/my-bucket/myfile.txt ./
|
||||
|
||||
# List objects
|
||||
mc ls myminio/my-bucket
|
||||
|
||||
# Remove object
|
||||
mc rm myminio/my-bucket/myfile.txt
|
||||
```
|
||||
|
||||
### **Python SDK (boto3)**
|
||||
|
||||
```python
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
|
||||
# Configure S3 client
|
||||
s3 = boto3.client(
|
||||
's3',
|
||||
endpoint_url='https://ai.sriphat.com/minio',
|
||||
aws_access_key_id='minioadmin',
|
||||
aws_secret_access_key='your-password',
|
||||
config=Config(signature_version='s3v4'),
|
||||
region_name='ap-southeast-1'
|
||||
)
|
||||
|
||||
# List buckets
|
||||
response = s3.list_buckets()
|
||||
for bucket in response['Buckets']:
|
||||
print(bucket['Name'])
|
||||
|
||||
# Upload file
|
||||
s3.upload_file('myfile.txt', 'my-bucket', 'myfile.txt')
|
||||
|
||||
# Download file
|
||||
s3.download_file('my-bucket', 'myfile.txt', 'downloaded.txt')
|
||||
|
||||
# List objects
|
||||
response = s3.list_objects_v2(Bucket='my-bucket')
|
||||
for obj in response.get('Contents', []):
|
||||
print(obj['Key'])
|
||||
```
|
||||
|
||||
### **AWS CLI**
|
||||
|
||||
```bash
|
||||
# Configure AWS CLI
|
||||
aws configure set aws_access_key_id minioadmin
|
||||
aws configure set aws_secret_access_key your-password
|
||||
aws configure set region ap-southeast-1
|
||||
|
||||
# List buckets
|
||||
aws --endpoint-url https://ai.sriphat.com/minio s3 ls
|
||||
|
||||
# Create bucket
|
||||
aws --endpoint-url https://ai.sriphat.com/minio s3 mb s3://my-bucket
|
||||
|
||||
# Upload file
|
||||
aws --endpoint-url https://ai.sriphat.com/minio s3 cp myfile.txt s3://my-bucket/
|
||||
|
||||
# Download file
|
||||
aws --endpoint-url https://ai.sriphat.com/minio s3 cp s3://my-bucket/myfile.txt ./
|
||||
|
||||
# Sync directory
|
||||
aws --endpoint-url https://ai.sriphat.com/minio s3 sync ./mydir s3://my-bucket/mydir/
|
||||
```
|
||||
|
||||
## 🔧 Configuration
|
||||
|
||||
### **Environment Variables**
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `MINIO_ROOT_USER` | Root username | minioadmin |
|
||||
| `MINIO_ROOT_PASSWORD` | Root password | - |
|
||||
| `MINIO_API_PORT` | API port | 9000 |
|
||||
| `MINIO_CONSOLE_PORT` | Console port | 9001 |
|
||||
| `MINIO_SERVER_URL` | API endpoint URL | - |
|
||||
| `MINIO_BROWSER_REDIRECT_URL` | Console URL | - |
|
||||
| `MINIO_REGION` | Default region | ap-southeast-1 |
|
||||
|
||||
### **Keycloak Integration**
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `MINIO_IDENTITY_OPENID_CONFIG_URL` | Keycloak OIDC config URL |
|
||||
| `MINIO_IDENTITY_OPENID_CLIENT_ID` | Client ID in Keycloak |
|
||||
| `MINIO_IDENTITY_OPENID_CLIENT_SECRET` | Client secret |
|
||||
| `MINIO_IDENTITY_OPENID_CLAIM_NAME` | Policy claim name |
|
||||
| `MINIO_IDENTITY_OPENID_SCOPES` | OIDC scopes |
|
||||
|
||||
### **Storage**
|
||||
|
||||
**Persistent Data:**
|
||||
```
|
||||
07-minio/data/ # Object storage data
|
||||
07-minio/certs/ # SSL certificates (optional)
|
||||
```
|
||||
|
||||
**Volume Mounts:**
|
||||
```yaml
|
||||
volumes:
|
||||
- ./data:/data # Storage data
|
||||
- ./certs:/root/.minio/certs:ro # SSL certs
|
||||
```
|
||||
|
||||
## 🔒 Security
|
||||
|
||||
### **1. Strong Passwords**
|
||||
|
||||
```bash
|
||||
# Generate strong password
|
||||
openssl rand -base64 32
|
||||
|
||||
# Update .env
|
||||
MINIO_ROOT_PASSWORD=generated-password-here
|
||||
```
|
||||
|
||||
### **2. Network Security**
|
||||
|
||||
```bash
|
||||
# Firewall rules (if needed)
|
||||
sudo ufw allow from 192.168.100.0/24 to any port 9000
|
||||
sudo ufw allow from 192.168.100.0/24 to any port 9001
|
||||
```
|
||||
|
||||
### **3. HTTPS Only**
|
||||
|
||||
- Always use HTTPS in production
|
||||
- Configure SSL certificates in Nginx
|
||||
- Set `MINIO_SERVER_URL` and `MINIO_BROWSER_REDIRECT_URL` to HTTPS
|
||||
|
||||
### **4. Access Policies**
|
||||
|
||||
```bash
|
||||
# Create read-only policy
|
||||
mc admin policy create myminio readonly-policy readonly-policy.json
|
||||
|
||||
# Assign policy to user
|
||||
mc admin policy attach myminio readonly-policy --user=username
|
||||
```
|
||||
|
||||
### **5. Bucket Policies**
|
||||
|
||||
```bash
|
||||
# Set bucket policy (public read)
|
||||
mc anonymous set download myminio/public-bucket
|
||||
|
||||
# Set bucket policy (private)
|
||||
mc anonymous set none myminio/private-bucket
|
||||
```
|
||||
|
||||
## 📊 Monitoring
|
||||
|
||||
### **Health Check**
|
||||
|
||||
```bash
|
||||
# Check MinIO health
|
||||
curl -k https://ai.sriphat.com/minio/health/live
|
||||
|
||||
# Check from container
|
||||
docker exec minio curl -f http://localhost:9000/minio/health/live
|
||||
```
|
||||
|
||||
### **Logs**
|
||||
|
||||
```bash
|
||||
# View logs
|
||||
docker logs minio -f
|
||||
|
||||
# View last 100 lines
|
||||
docker logs minio --tail 100
|
||||
|
||||
# Export logs
|
||||
docker logs minio > minio.log
|
||||
```
|
||||
|
||||
### **Metrics**
|
||||
|
||||
```bash
|
||||
# View server info
|
||||
mc admin info myminio
|
||||
|
||||
# View server stats
|
||||
mc admin prometheus metrics myminio
|
||||
```
|
||||
|
||||
### **Disk Usage**
|
||||
|
||||
```bash
|
||||
# Check disk usage
|
||||
mc admin info myminio
|
||||
|
||||
# Check bucket size
|
||||
mc du myminio/my-bucket
|
||||
```
|
||||
|
||||
## 🐛 Troubleshooting
|
||||
|
||||
### **Issue: Cannot access MinIO Console**
|
||||
|
||||
**Check:**
|
||||
```bash
|
||||
# Verify container is running
|
||||
docker ps | grep minio
|
||||
|
||||
# Check logs
|
||||
docker logs minio
|
||||
|
||||
# Test direct access
|
||||
curl http://192.168.100.9:9001
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Ensure container is running: `docker compose up -d`
|
||||
- Check firewall rules
|
||||
- Verify Nginx configuration
|
||||
|
||||
### **Issue: SSO login not working**
|
||||
|
||||
**Check:**
|
||||
```bash
|
||||
# Verify Keycloak config
|
||||
docker exec minio printenv | grep MINIO_IDENTITY_OPENID
|
||||
|
||||
# Test Keycloak connectivity
|
||||
docker exec minio curl -k https://ai.sriphat.com/keycloak/realms/sriphat/.well-known/openid-configuration
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Verify all Keycloak environment variables are set
|
||||
- Check client secret is correct
|
||||
- Ensure redirect URIs match in Keycloak
|
||||
- See `KEYCLOAK_INTEGRATION.md` for detailed troubleshooting
|
||||
|
||||
### **Issue: Upload fails**
|
||||
|
||||
**Check:**
|
||||
```bash
|
||||
# Check disk space
|
||||
df -h
|
||||
|
||||
# Check permissions
|
||||
ls -la data/
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Ensure sufficient disk space
|
||||
- Check directory permissions: `chmod 755 data/`
|
||||
- Increase `client_max_body_size` in Nginx
|
||||
|
||||
### **Issue: S3 API connection refused**
|
||||
|
||||
**Check:**
|
||||
```bash
|
||||
# Test API endpoint
|
||||
curl -k https://ai.sriphat.com/minio/
|
||||
|
||||
# Test direct connection
|
||||
curl http://192.168.100.9:9000/
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
- Verify `MINIO_SERVER_URL` is set correctly
|
||||
- Check Nginx proxy configuration
|
||||
- Ensure port 9000 is accessible
|
||||
|
||||
## 🔄 Maintenance
|
||||
|
||||
### **Backup**
|
||||
|
||||
```bash
|
||||
# Backup data directory
|
||||
tar -czf minio-backup-$(date +%Y%m%d).tar.gz data/
|
||||
|
||||
# Backup to remote location
|
||||
rsync -avz data/ user@backup-server:/backups/minio/
|
||||
```
|
||||
|
||||
### **Update MinIO**
|
||||
|
||||
```bash
|
||||
# Pull latest image
|
||||
docker compose pull
|
||||
|
||||
# Restart with new image
|
||||
docker compose up -d
|
||||
|
||||
# Verify version
|
||||
docker exec minio minio --version
|
||||
```
|
||||
|
||||
### **Restore**
|
||||
|
||||
```bash
|
||||
# Stop MinIO
|
||||
docker compose down
|
||||
|
||||
# Restore data
|
||||
tar -xzf minio-backup-20260325.tar.gz
|
||||
|
||||
# Start MinIO
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## 📚 Documentation
|
||||
|
||||
- **MinIO Official Docs**: https://min.io/docs/minio/linux/
|
||||
- **S3 API Reference**: https://docs.aws.amazon.com/AmazonS3/latest/API/
|
||||
- **Keycloak Integration**: See `KEYCLOAK_INTEGRATION.md`
|
||||
- **Nginx Configuration**: See `nginx-minio.conf`
|
||||
|
||||
## 🎯 Use Cases
|
||||
|
||||
### **1. Data Lake Storage**
|
||||
- Store raw data files (CSV, JSON, Parquet)
|
||||
- Integrate with Spark, Pandas, Dask
|
||||
- Version control for datasets
|
||||
|
||||
### **2. Backup Storage**
|
||||
- Database backups
|
||||
- Application backups
|
||||
- Log archival
|
||||
|
||||
### **3. Media Storage**
|
||||
- Images, videos, documents
|
||||
- CDN integration
|
||||
- Static website hosting
|
||||
|
||||
### **4. ML/AI Workflows**
|
||||
- Model storage
|
||||
- Training data storage
|
||||
- Experiment artifacts
|
||||
|
||||
### **5. Application Storage**
|
||||
- User uploads
|
||||
- Generated reports
|
||||
- Temporary files
|
||||
|
||||
## 🎉 Summary
|
||||
|
||||
**What You Have:**
|
||||
- ✅ MinIO object storage service
|
||||
- ✅ Persistent storage with volume mounts
|
||||
- ✅ HTTPS access via Nginx reverse proxy
|
||||
- ✅ Keycloak SSO integration ready
|
||||
- ✅ S3-compatible API
|
||||
- ✅ Web console for management
|
||||
- ✅ Health checks and monitoring
|
||||
|
||||
**Access:**
|
||||
- Console: `https://ai.sriphat.com/minio-console`
|
||||
- API: `https://ai.sriphat.com/minio`
|
||||
|
||||
**Next Steps:**
|
||||
1. Configure `.env` file
|
||||
2. Start MinIO: `docker compose up -d`
|
||||
3. Setup Keycloak integration (optional)
|
||||
4. Configure Nginx reverse proxy
|
||||
5. Create buckets and start using!
|
||||
|
||||
For detailed Keycloak SSO setup, see `KEYCLOAK_INTEGRATION.md` 🚀
|
||||
50
07-minio/docker-compose.yml
Normal file
50
07-minio/docker-compose.yml
Normal file
@@ -0,0 +1,50 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: minio
|
||||
command: server /data --console-address ":9001"
|
||||
ports:
|
||||
- "${MINIO_API_PORT:-9000}:9000"
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001"
|
||||
environment:
|
||||
# MinIO credentials
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
|
||||
# Server settings
|
||||
MINIO_SERVER_URL: ${MINIO_SERVER_URL:-https://ai.sriphat.com/minio}
|
||||
MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL:-https://ai.sriphat.com/minio-console}
|
||||
|
||||
# Region
|
||||
MINIO_REGION: ${MINIO_REGION:-ap-southeast-1}
|
||||
|
||||
# Identity OpenID (Keycloak)
|
||||
MINIO_IDENTITY_OPENID_CONFIG_URL: ${MINIO_IDENTITY_OPENID_CONFIG_URL}
|
||||
MINIO_IDENTITY_OPENID_CLIENT_ID: ${MINIO_IDENTITY_OPENID_CLIENT_ID}
|
||||
MINIO_IDENTITY_OPENID_CLIENT_SECRET: ${MINIO_IDENTITY_OPENID_CLIENT_SECRET}
|
||||
MINIO_IDENTITY_OPENID_CLAIM_NAME: ${MINIO_IDENTITY_OPENID_CLAIM_NAME:-policy}
|
||||
MINIO_IDENTITY_OPENID_SCOPES: ${MINIO_IDENTITY_OPENID_SCOPES:-openid,profile,email}
|
||||
MINIO_IDENTITY_OPENID_REDIRECT_URI: ${MINIO_IDENTITY_OPENID_REDIRECT_URI}
|
||||
|
||||
# Timezone
|
||||
TZ: ${TZ:-Asia/Bangkok}
|
||||
volumes:
|
||||
# Persistent storage
|
||||
- ./data:/data
|
||||
# SSL certificates (if using direct HTTPS)
|
||||
- ./certs:/root/.minio/certs:ro
|
||||
networks:
|
||||
- shared_data_network
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
|
||||
networks:
|
||||
shared_data_network:
|
||||
external: true
|
||||
104
07-minio/nginx-minio.conf
Normal file
104
07-minio/nginx-minio.conf
Normal file
@@ -0,0 +1,104 @@
|
||||
# MinIO Nginx Configuration
|
||||
# For use with Nginx Proxy Manager or standalone Nginx
|
||||
# This configuration provides HTTPS access to MinIO API and Console
|
||||
|
||||
# ============================================================================
|
||||
# MinIO S3 API - Port 9000
|
||||
# Subpath: /minio
|
||||
# ============================================================================
|
||||
location /minio/ {
|
||||
# Rewrite path to remove /minio prefix
|
||||
rewrite ^/minio/(.*) /$1 break;
|
||||
|
||||
# Forward to MinIO API
|
||||
proxy_pass http://192.168.100.9:9000;
|
||||
|
||||
# Preserve headers
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
|
||||
# Required for MinIO
|
||||
proxy_set_header X-NginX-Proxy true;
|
||||
|
||||
# Disable buffering for large uploads
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
# Timeouts for large file uploads/downloads
|
||||
proxy_connect_timeout 300;
|
||||
proxy_send_timeout 300;
|
||||
proxy_read_timeout 300;
|
||||
send_timeout 300;
|
||||
|
||||
# Max upload size (adjust as needed)
|
||||
client_max_body_size 0;
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# MinIO Console (Web UI) - Port 9001
|
||||
# Subpath: /minio-console
|
||||
# ============================================================================
|
||||
location /minio-console/ {
|
||||
# Forward to MinIO Console
|
||||
proxy_pass http://192.168.100.9:9001/;
|
||||
|
||||
# Preserve headers
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
|
||||
# WebSocket support for real-time updates
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
|
||||
# Disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
# Timeouts
|
||||
proxy_connect_timeout 300;
|
||||
proxy_send_timeout 300;
|
||||
proxy_read_timeout 300;
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# MinIO Console Assets
|
||||
# ============================================================================
|
||||
location /minio-console/assets/ {
|
||||
proxy_pass http://192.168.100.9:9001/assets/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_valid 200 1d;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# MinIO Console API
|
||||
# ============================================================================
|
||||
location /minio-console/api/ {
|
||||
proxy_pass http://192.168.100.9:9001/api/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# WebSocket support
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_buffering off;
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# MinIO Health Check
|
||||
# ============================================================================
|
||||
location /minio/health {
|
||||
proxy_pass http://192.168.100.9:9000/minio/health;
|
||||
proxy_set_header Host $host;
|
||||
access_log off;
|
||||
}
|
||||
Reference in New Issue
Block a user