# Warehouse: DuckDB compute over an Iceberg REST catalog. # # DuckDB runs in memory (jdbc:duckdb: with no path); the tables live in the # catalog, attached under WAREHOUSE_CATALOG. Iceberg has no sequences or keys, # so ROW_ID is computed per insert, and each statement commits on its own. # Omit s3_key_id/s3_secret to use the ambient credential chain. The iceberg and # httpfs extensions are installed on first connection. # # Each setting is marked `# variable` or `# value`. A variable is looked up in # the secrets source (here: the process environment); when it is not set, the # text itself is used, so `Profile: dev` is simply dev. A secret must always # be a variable that is set. `etl-craft doctor` lists every value used as # written that looks like a variable name. # # Complete and loadable as written: tests/unit/test_config_examples.py loads every # file in this directory, for every profile it declares. Secrets: Source_type: environment # value — variables below are read from the process environment Profile: ETL_CRAFT_PROFILE # variable — dev | sit | uat | prod, set in each environment Orchestration: Mode: remote # value — an orchestrator runs each task from generate-yml's DAGs Name: Airflow # value — informational: the YAML DAG is the same; convert it with your own scripts Task_timeout_seconds: 21600 # value Max_parallel_tasks: 8 # value Enforce_sla: false # value — true: email when a run misses its SLA_IN_HOURS dev: Global_dag: false # value Catchup: false # value Tags: [analytics] # value Retries: 1 # value Retry_delay_minutes: 5 # value Depends_on_past: false # value Email_on_failure: false # value Allow_schedule: false # value sit: Global_dag: false # value Catchup: false # value Tags: [analytics] # value Retries: 1 # value Retry_delay_minutes: 5 # value Depends_on_past: false # value Email_on_failure: false # value Allow_schedule: false # value Email: host: EMAIL_HOST # variable port: EMAIL_PORT # variable from_address: EMAIL_FROM # variable auth_mode: EMAIL_AUTH_MODE # variable user: EMAIL_USER # variable tls_mode: EMAIL_TLS_MODE # variable — starttls | ssl | none secret: EMAIL_SECRET # variable, required: a secret is never used as written uat: Global_dag: false # value Catchup: false # value Tags: [analytics] # value Retries: 1 # value Retry_delay_minutes: 5 # value Depends_on_past: false # value Email_on_failure: false # value Allow_schedule: false # value Email: host: EMAIL_HOST # variable port: EMAIL_PORT # variable from_address: EMAIL_FROM # variable auth_mode: EMAIL_AUTH_MODE # variable user: EMAIL_USER # variable tls_mode: EMAIL_TLS_MODE # variable — starttls | ssl | none secret: EMAIL_SECRET # variable, required: a secret is never used as written prod: Global_dag: true # value Catchup: false # value Tags: [analytics] # value Retries: 1 # value Retry_delay_minutes: 5 # value Depends_on_past: false # value Email_on_failure: true # value Allow_schedule: true # value Email_recipients: [data-alerts@example.com] # value Email: host: EMAIL_HOST # variable port: EMAIL_PORT # variable from_address: EMAIL_FROM # variable auth_mode: EMAIL_AUTH_MODE # variable user: EMAIL_USER # variable tls_mode: EMAIL_TLS_MODE # variable — starttls | ssl | none secret: EMAIL_SECRET # variable, required: a secret is never used as written Engine: Name: Postgres # value dev: jdbc_url: ENGINE_JDBC_URL # variable schema: etl_craft # value — must already exist; the engine never creates it user: ENGINE_USER # variable auth_mode: ENGINE_AUTH_MODE # variable secret: ENGINE_SECRET # variable, required: a secret is never used as written sit: jdbc_url: ENGINE_JDBC_URL # variable schema: etl_craft # value — must already exist; the engine never creates it user: ENGINE_USER # variable auth_mode: ENGINE_AUTH_MODE # variable secret: ENGINE_SECRET # variable, required: a secret is never used as written uat: jdbc_url: ENGINE_JDBC_URL # variable schema: etl_craft # value — must already exist; the engine never creates it user: ENGINE_USER # variable auth_mode: ENGINE_AUTH_MODE # variable secret: ENGINE_SECRET # variable, required: a secret is never used as written prod: jdbc_url: ENGINE_JDBC_URL # variable schema: etl_craft # value — must already exist; the engine never creates it user: ENGINE_USER # variable auth_mode: ENGINE_AUTH_MODE # variable secret: ENGINE_SECRET # variable, required: a secret is never used as written Warehouse: Name: DuckDB # value Table_format: iceberg # value dev: jdbc_url: "jdbc:duckdb:" # value — in memory — the data lives in Iceberg (quoted: it ends in a colon) schema: etl_craft # value — must already exist; cloning writes here catalog: WAREHOUSE_CATALOG # variable — the name it is attached as, e.g. lake catalog_uri: WAREHOUSE_CATALOG_URI # variable — the REST catalog endpoint iceberg_warehouse: WAREHOUSE_ICEBERG_WAREHOUSE # variable — e.g. s3://warehouse/ s3_endpoint: WAREHOUSE_S3_ENDPOINT # variable s3_region: WAREHOUSE_S3_REGION # variable s3_url_style: WAREHOUSE_S3_URL_STYLE # variable — path | vhost s3_use_ssl: WAREHOUSE_S3_USE_SSL # variable s3_key_id: WAREHOUSE_S3_KEY_ID # variable s3_secret: WAREHOUSE_S3_SECRET # variable, required: a secret is never used as written sit: jdbc_url: "jdbc:duckdb:" # value — in memory — the data lives in Iceberg (quoted: it ends in a colon) schema: etl_craft # value — must already exist; cloning writes here catalog: WAREHOUSE_CATALOG # variable — the name it is attached as, e.g. lake catalog_uri: WAREHOUSE_CATALOG_URI # variable — the REST catalog endpoint iceberg_warehouse: WAREHOUSE_ICEBERG_WAREHOUSE # variable — e.g. s3://warehouse/ s3_endpoint: WAREHOUSE_S3_ENDPOINT # variable s3_region: WAREHOUSE_S3_REGION # variable s3_url_style: WAREHOUSE_S3_URL_STYLE # variable — path | vhost s3_use_ssl: WAREHOUSE_S3_USE_SSL # variable s3_key_id: WAREHOUSE_S3_KEY_ID # variable s3_secret: WAREHOUSE_S3_SECRET # variable, required: a secret is never used as written uat: jdbc_url: "jdbc:duckdb:" # value — in memory — the data lives in Iceberg (quoted: it ends in a colon) schema: etl_craft # value — must already exist; cloning writes here catalog: WAREHOUSE_CATALOG # variable — the name it is attached as, e.g. lake catalog_uri: WAREHOUSE_CATALOG_URI # variable — the REST catalog endpoint iceberg_warehouse: WAREHOUSE_ICEBERG_WAREHOUSE # variable — e.g. s3://warehouse/ s3_endpoint: WAREHOUSE_S3_ENDPOINT # variable s3_region: WAREHOUSE_S3_REGION # variable s3_url_style: WAREHOUSE_S3_URL_STYLE # variable — path | vhost s3_use_ssl: WAREHOUSE_S3_USE_SSL # variable s3_key_id: WAREHOUSE_S3_KEY_ID # variable s3_secret: WAREHOUSE_S3_SECRET # variable, required: a secret is never used as written prod: jdbc_url: "jdbc:duckdb:" # value — in memory — the data lives in Iceberg (quoted: it ends in a colon) schema: etl_craft # value — must already exist; cloning writes here catalog: WAREHOUSE_CATALOG # variable — the name it is attached as, e.g. lake catalog_uri: WAREHOUSE_CATALOG_URI # variable — the REST catalog endpoint iceberg_warehouse: WAREHOUSE_ICEBERG_WAREHOUSE # variable — e.g. s3://warehouse/ s3_endpoint: WAREHOUSE_S3_ENDPOINT # variable s3_region: WAREHOUSE_S3_REGION # variable s3_url_style: WAREHOUSE_S3_URL_STYLE # variable — path | vhost s3_use_ssl: WAREHOUSE_S3_USE_SSL # variable s3_key_id: WAREHOUSE_S3_KEY_ID # variable s3_secret: WAREHOUSE_S3_SECRET # variable, required: a secret is never used as written