update config for limit resouce size

05-airflow/config/airflow.cfg

@@ -40,7 +40,7 @@ might_contain_dag_callable = airflow.utils.file.might_contain_dag_via_default_he
 #
 # Variable: AIRFLOW__CORE__DEFAULT_TIMEZONE
 #
-default_timezone = utc
+default_timezone = Asia/Bangkok
 
 # The executor class that airflow should use. Choices include
 # ``LocalExecutor``, ``CeleryExecutor``,
@@ -90,7 +90,7 @@ simple_auth_manager_all_admins = False
 #
 # Variable: AIRFLOW__CORE__PARALLELISM
 #
-parallelism = 8
+parallelism = 2
 
 # The maximum number of task instances allowed to run concurrently in each dag run.
 # This is also configurable per-dag with ``max_active_tasks``,
@@ -115,7 +115,7 @@ dags_are_paused_at_creation = True
 #
 # Variable: AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG
 #
-max_active_runs_per_dag = 16
+max_active_runs_per_dag = 1
 
 # (experimental) The maximum number of consecutive DAG failures before DAG is automatically paused.
 # This is also configurable per DAG level with ``max_consecutive_failed_dag_runs``,
@@ -2166,7 +2166,7 @@ refresh_interval = 300
 #
 # Variable: AIRFLOW__DAG_PROCESSOR__PARSING_PROCESSES
 #
-parsing_processes = 2
+parsing_processes = 1
 
 # One of ``modified_time``, ``random_seeded_by_host`` and ``alphabetical``.
 # The DAG processor will list and sort the dag files to decide the parsing order.
@@ -2193,7 +2193,9 @@ max_callbacks_per_loop = 20
 #
 # Variable: AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL
 #
-min_file_process_interval = 30
+min_file_process_interval = 90
+
+dag_dir_list_interval = 90
 
 # How long (in seconds) to wait after we have re-parsed a DAG file before deactivating stale
 # DAGs (DAGs which are no longer present in the expected files). The reason why we need
@@ -2491,7 +2493,7 @@ flower_basic_auth =
 #
 # Variable: AIRFLOW__CELERY__SYNC_PARALLELISM
 #
-sync_parallelism = 0
+sync_parallelism = 2
 
 # Import path for celery configuration options
 #

05-airflow/docker-compose.yaml

@@ -64,7 +64,7 @@ x-airflow-common:
     AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
     AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY:-}
     AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
-    AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES:-'false'}
+    AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES:-False}
     AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/'
     # yamllint disable rule:line-length
     # Use simple http server on scheduler for health checks
@@ -76,18 +76,22 @@ x-airflow-common:
     _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
     # The following line can be used to set a custom config file, stored in the local config folder
     AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'
+    AIRFLOW__WEBSERVER__BASE_URL: ${AIRFLOW__WEBSERVER__BASE_URL:-https://ai.sriphat.com/airflow}
+    AIRFLOW__API__BASE_URL:  ${AIRFLOW__WEBSERVER__BASE_URL:-https://ai.sriphat.com/airflow}
+    AIRFLOW__WEBSERVER__WEB_SERVER_PORT: ${AIRFLOW__WEBSERVER__WEB_SERVER_PORT:-8080}
   volumes:
     - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags
     - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
     - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config
     - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins
   user: "${AIRFLOW_UID:-50000}:0"
-  depends_on:
-    &airflow-common-depends-on
-    #airflow-base:
+  x-depends_on:
+   &airflow-common-depends-on
+   {}
+    # airflow-base:
     #  condition: service_completed_successfully
-    redis:
-      condition: service_healthy
+    # redis:
+    #  condition: service_healthy
   networks:
     - shared_data_network
 
@@ -114,19 +118,19 @@ services:
       # start_period: 5s
     # restart: always
 
-  redis:
-    # Redis is limited to 7.2-bookworm due to licencing change
-    # https://redis.io/blog/redis-adopts-dual-source-available-licensing/
-    image: redis:7.2-bookworm
-    expose:
-      - 6379
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 10s
-      timeout: 30s
-      retries: 50
-      start_period: 30s
-    restart: always
+  # redis:
+  #   # Redis is limited to 7.2-bookworm due to licencing change
+  #   # https://redis.io/blog/redis-adopts-dual-source-available-licensing/
+  #   image: redis:7.2-bookworm
+  #   expose:
+  #     - 6379
+  #   healthcheck:
+  #     test: ["CMD", "redis-cli", "ping"]
+  #     interval: 10s
+  #     timeout: 30s
+  #     retries: 50
+  #     start_period: 30s
+  #   restart: always
 
   airflow-apiserver:
     <<: *airflow-common