# DataOps Crawler module configuration without project integration.
# Use this approach when deploying crawlers independently of a
# DataOps project. Infrastructure resources (KMS key, S3 bucket,
# IAM roles, SNS topic, security configuration) must be provided
# directly rather than autowired from a project.

# (Optional) KMS key ARN for encrypting DataOps resources and data.
kmsArn: arn:{{partition}}:kms:{{region}}:{{account}}:key/test-key-id
# (Optional) Glue security configuration name for crawler encryption
# (at rest, in transit, CloudWatch logs).
securityConfigurationName: test-security-config
# (Optional) S3 bucket name for project storage (scripts, artifacts,
# temp files).
bucketName: test-crawler-bucket
# (Optional) IAM role ARN for deployment operations and resource
# management.
deploymentRoleArn: arn:{{partition}}:iam::{{account}}:role/test-deploy-role
# (Optional) SNS topic ARN for job notifications and workflow alerts.
notificationTopicArn: arn:{{partition}}:sns:{{region}}:{{account}}:test-topic

# Map of crawler names to Glue crawler definitions for data source
# discovery and cataloging.
crawlers:
  test-crawler:
    # ARN of the execution role
    # Often created by the Roles module.
    # Example SSM: ssm:/{{org}}/{{domain}}/<roles_module_name>/role/<role_name>/arn
    executionRoleArn: some-arn
    # Name of the database to crawl
    databaseName: project:databaseName/example-database
    # Description for the crawler
    description: Example for a Crawler
    # Crawler targets specifying data sources to crawl
    targets:
      # S3 targets for data lake object discovery
      s3Targets:
        - # Path to the S3 target
          path: s3://some-s3-bucket/path/to/crawler/target
    # (Optional) Custom classifier names
    classifiers:
      - project:classifiers/classifierCsv
    # (Optional) Recrawl behavior
    recrawlBehavior: CRAWL_NEW_FOLDERS_ONLY
    # (Optional) Extra crawler configuration
    extraConfiguration:
      Version: 1
      CrawlerOutput:
        Partitions:
          AddOrUpdateBehavior: InheritFromTable