Postgres

Read Config

postgres_depot_read.yml

# creds depends upon the env refer to default schema public
version: v1
name: postgres-write-01
type: workflow
tags:
  - bq
  - dataset
description: This job read and write data from to postgres
title: Read Write Postgres
workflow:
  dag:
    - name: write-postgres-01
      title: Write Postgres
      description: This job reads data from postgres
      spec:
        tags:
          - Connect
        stack: flare:6.0
        compute: runnable-default

        flare:
          job:
            explain: true
            inputs:
              - name: city_connect
                dataset: dataos://thirdparty01:none/city
                format: csv
                schemaPath: dataos://thirdparty01:none/schemas/avsc/city.avsc
            logLevel: INFO
            outputs:
              - name: input
                dataset: dataos://sanitypostgres:public/postgres_write_12?acl=rw
                driver: org.postgresql.Driver
                format: jdbc
                options:
                  saveMode: overwrite
                description: Data set from lakehouse
                tags:
                  - Connect
                title: Postgres Dataset
            steps:
              - sequence:
                  - name: input
                    doc: Pick all columns from cities and add version as yyyyMMddHHmm formatted
                      timestamp.
                    sql: SELECT * FROM city_connect limit 10        

Write Config

postgres_depot_write.yml

# creds depends upon the env refer to default schema public
version: v1
name: postgres-write-01
type: workflow
tags:
  - bq
  - dataset
description: This job read and write data from to postgres
title: Read Write Postgres
workflow:
  dag:
    - name: write-postgres-01
      title: Write Postgres
      description: This job write data to postgres
      spec:
        tags:
          - Connect
        stack: flare:6.0
        compute: runnable-default

        flare:
          job:
            explain: true
            inputs:
              - name: city_connect
                dataset: dataos://thirdparty01:none/city
                format: csv
                schemaPath: dataos://thirdparty01:none/schemas/avsc/city.avsc
            logLevel: INFO
            outputs:
              - name: input
                dataset: dataos://sanitypostgres:public/postgres_write_12?acl=rw
                driver: org.postgresql.Driver
                format: jdbc
                options:
                  saveMode: overwrite
                description: Data set from Icebase
                tags:
                  - Connect
                title: Postgres Dataset
            steps:
              - sequence:
                  - name: input
                    doc: Pick all columns from cities and add version as yyyyMMddHHmm formatted
                      timestamp.
                    sql: SELECT * FROM city_connect limit 10