Opensearch

Read Config

opensearch_depot_read.yml

version: v1
name: read-opensearch-01
type: workflow
tags:
  - opensearch
  - read
description: this jobs reads data from opensearch and writes to lakehouse
workflow:
  dag:
    - name: opensearch-read-01
      title: read data from opensearch
      description: read data from opensearch
      spec:
        tags:
          - Connect
        stack: flare:6.0
        compute: runnable-default
        flare:
          job:
            explain: true
            inputs:
              - name: input
                dataset: dataos://sanityopensearch:default/opensearch_write_12
                format: opensearch
                options:
                  'opensearch.nodes.wan.only': 'true'
            logLevel: INFO
            outputs:
              - name: finalDf
                dataset: dataos://lakehouse:smoketest/opensearch_read_12?acl=rw
                format: iceberg
                options:
                  saveMode: append

            steps:
              - sequence:
                - name: finalDf
                  sql: SELECT * FROM input limit 10

Write Config

opensearch_depot_write.yml

version: v1
name: write-opensearch-01
type: workflow
tags:
  - opensearch
  - write
description: this jobs reads data from thirdparty and writes to opensearch
workflow:
  dag:
    - name: opensearch-write-01
      title: write data to opensearch
      description: write data to opensearch
      spec:
        tags:
          - Connect
        stack: flare:6.0
        compute: runnable-default
        flare:
          job:
            explain: true
            inputs:
              - name: input
                dataset: dataos://thirdparty01:none/city
                format: csv
                schemaPath: dataos://thirdparty01:none/schemas/avsc/city.avsc
                # isStream: false
            logLevel: INFO
            outputs:
              - name: finalDf
                dataset: dataos://sanityopensearch:default/opensearch_write_12?acl=rw
                format: opensearch
                options:
                  extraOptions:
                    'opensearch.nodes.wan.only': 'true'
            steps:
              - sequence:
                - name: finalDf
                  sql: SELECT * FROM input limit 10