MySQL

Read Config

mysql_depot_read.yml

version: v1
name: cnt-city-demo-mysql-01
type: workflow
tags:
  - Connect
  - read
description: Jobs writes data to mssql and reads from it
workflow:
  title: Read MSSQL
  dag:
    - name: city-mysql-read
      title: Reading data and writing to dataos
      description: This job writes data to dataos
      spec:
        tags:
          - Connect
          - read
        stack: flare:5.0
        compute: runnable-default
        stackSpec:
          job:
            explain: true
            inputs:
              - name: city_connect
                dataset: dataos://sanitymysqlalok01:dataosmysql/city01
                options:
                  driver: com.mysql.cj.jdbc.Driver
            logLevel: INFO
            outputs:
              - name: cities
                dataset: dataos://icebase:sample/city_from_mysql?acl=rw
                format: Iceberg
                options:
                  saveMode: overwrite
                  iceberg:
                    properties:
                      write.format.default: parquet
                      write.metadatacompression-codec: gzip
                description: City data ingested from external csv
                title: City Source Data

            steps:
              - sequence:
                - name: cities
                  sql: SELECT * FROM city_connect

Write Config

mysql_depot_write.yml

version: v1
name: cnt-city-demo-mysql-01
type: workflow
tags:
  - Connect
  - City
description: The job ingests city data from dropzone into raw zone
workflow:
  title: Connect City
  dag:
    - name: city-mysql-01
      title: City Dimension Ingester
      description: The job ingests city data from dropzone into raw zone
      spec:
        tags:
          - Connect
          - City
        stack: flare:5.0
        compute: runnable-default
        stackSpec:
          job:
            explain: true
            inputs:
              - name: city_connect
                dataset: dataos://icebase:retail/city
                format: iceberg
            logLevel: INFO
            outputs:
              - name: cities
                dataset: dataos://mysql01:dataosmysql/city01?acl=rw
                driver: com.mysql.cj.jdbc.Driver
                format: jdbc
                description: City data ingested from external csv
                options:
                  saveMode: overwrite
                  extraOptions:
                    "rewriteBatchedStatements": "true"
                isStream: false
                tags:
                  - Connect
                  - City
                title: City Source Data
            steps:
              - sequence:
                  - name: cities
                    doc: Pick all columns from cities and add version as yyyyMMddHHmm formatted
                      timestamp.
                    sql: SELECT * FROM city_connect LIMIT 1000