Postgres

Read Config

postgres_depot_read.yml

version: v1
name: postgres-read-write
type: workflow
tags:
  - bq
  - dataset
description: This job read and write data from to postgres
title: Read Write Postgres
workflow:
  dag:
    - name: read-postgres
      title: Read Postgres
      description: This job read data from postgres
      spec:
        tags:
          - Connect
        stack: flare:5.0
        compute: runnable-default
        stackSpec:
          job:
            explain: true
            inputs:
              - name: input
                dataset: dataos://sanitypostgresalok02:public/postgres_write
                options:
                  driver: org.postgresql.Driver
            logLevel: INFO
            outputs:
              - name: input
                dataset: dataos://icebase:sanity/sanity_postgres?acl=rw
                format: Iceberg
                options:
                  saveMode: overwrite
                description: Data set from blender
                tags:
                  - Connect
                title: Postgres Dataset

Write Config

postgres_depot_write.yml

version: v1
name: postgres-read-write
type: workflow
tags:
  - bq
  - dataset
description: This job read and write data from to postgres
title: Read Write Postgres
workflow:
  dag:
    - name: write-postgres
      title: Write Postgres
      description: This job write data from postgres
      spec:
        tags:
          - Connect
        stack: flare:5.0
        compute: runnable-default
        stackSpec:
          job:
            explain: true
            inputs:
              - name: input
                dataset: dataos://thirdparty01:none/city
                format: csv
                schemaPath: dataos://thirdparty01:none/schemas/avsc/city.avsc
            logLevel: INFO
            outputs:
              - name: input
                dataset: dataos://sanitypostgresalok02:public/postgres_write?acl=rw
                driver: org.postgresql.Driver
                format: jdbc
                options:
                  saveMode: overwrite
                description: Data set from Icebase
                tags:
                  - Connect
                title: Postgres Dataset