YAML
####################################################################################################
# Example YAML for Tot Plugin Databricks 1.0 (Phase 1: metadata discovery and extraction) #
# #
# This file has all the technical properties that can be edited for the client. #
# These properties have a default value in the application, #
# so there is no necessity to configure them. #
# Only configure these properties if they are going to be different to the value of this file, #
# this will make your files more clean and readable. #
####################################################################################################
#########################
### Server properties ###
#########################
server:
# Port where service will be listening
port: 15021
#############################
### Tot Plugin properties ###
#############################
totplugin:
server:
# Tot server URL. It is a list because Tot can have more than one instance running.
urls:
- https://tot1server.anjanadata.local/totserver/
# Character used to build/interpret the logical path (path attribute). Default: "/"
# Use a different character if asset names may contain "/". Any character except ":" is allowed.
pathSeparator: "/"
# Separator used by Anjana to read the values of a list-type attribute (e.g. tags).
attributeSeparator: "_-"
# Rate limiting applied by the plugin against the Unity Catalog REST API
rate-limit:
requests-per-second: 10
timeout-seconds: 30
# Concurrency settings for listing and tag retrieval
concurrency:
listing-threads: 4
listing-batch-size: 8
tags-threads: 5
tags-batch-size: 10
################################
## Databricks connection properties
## name: Name of the connection instance
## aris: list of aris that define the functionalities of the instance (Phase 1: extract)
## technology.credential: specific connection/authentication properties of the instance
## ## host: Databricks workspace URL
## ## ## Azure: https://adb-xxxx.azuredatabricks.net
## ## ## AWS: https://xxxx.cloud.databricks.com
## ## ## GCP: https://xxxx.gcp.databricks.com
## ## authType: token strategy. Only "databricks-oauth" or "azure-sp" are allowed
## ## tenant: Azure Entra ID tenant id (only required when authType is "azure-sp")
## ## clientId: Service Principal id
## ## clientSecret: OAuth secret
## ## databricksCountRetry: max number of retries for transient errors
## ## databricksWaitRetry: base wait time (seconds) between retries
## ## maxPages: max pages per operation to avoid infinite pagination (default 1000)
## technology.mapping: subType assigned to each Unity Catalog asset type
################################
connection:
# --- Connection 1: native Databricks OAuth M2M (recommended, works on Azure/AWS/GCP) ---
- name: DBK-OAuth
aris:
- ari: "anja:totplugin:extract:/Databricks/databricks/dev-1/"
technology:
credential:
host: "https://adb-xxxx.azuredatabricks.net"
authType: "databricks-oauth"
clientId: "your-client-id"
clientSecret: "your-client-secret"
databricksCountRetry: 5
databricksWaitRetry: 5
maxPages: 1000
mapping:
catalog: DBK_CATALOG
schema: DBK_SCHEMA
table: DATASET
volume: DBK_VOLUME
# --- Connection 2: Azure Entra ID Service Principal (Azure-only alternative) ---
- name: DBK-AzureSP
aris:
- ari: "anja:totplugin:extract:/Databricks/databricks/dev-2/"
technology:
credential:
host: "https://adb-yyyy.azuredatabricks.net"
authType: "azure-sp"
tenant: "your-tenant-id"
clientId: "your-client-id"
clientSecret: "your-client-secret"
databricksCountRetry: 5
databricksWaitRetry: 5
maxPages: 1000
mapping:
catalog: DBK_CATALOG
schema: DBK_SCHEMA
table: DATASET
volume: DBK_VOLUME
#########################
### Logs properties ###
#########################
logging:
pattern:
# Pattern to convert logs into the convenient format for the client
console: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} [TOT-PLUGIN-DATABRICKS] %clr(${LOG_LEVEL_PATTERN:%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}"
# Log level per package. ERROR > WARN > INFO > DEBUG > TRACE
level:
root: info
com.anjana: DEBUG
#################################
### Hecate(Eureka) properties ###
#################################
eureka:
client:
serviceUrl:
# Url where Tot is listening
defaultZone: https://tot1server.anjanadata.local/totserver/