pyflare.sdk package

Subpackages

• pyflare.sdk.config package
  • Submodules
  • pyflare.sdk.config.constants module
    • get_log4j_properties_path()
    • get_spark_app_name()
  • pyflare.sdk.config.read_config module
    • ReadConfig
      • ReadConfig.collection()
      • ReadConfig.connection()
      • ReadConfig.dataset_absolute_path()
      • ReadConfig.dataset_name()
      • ReadConfig.depot_details
      • ReadConfig.depot_name()
      • ReadConfig.depot_type()
      • ReadConfig.driver
      • ReadConfig.extra_options
      • ReadConfig.format_resolver()
      • ReadConfig.io_format
      • ReadConfig.is_stream
      • ReadConfig.query
      • ReadConfig.spark_options
  • pyflare.sdk.config.write_config module
    • WriteConfig
      • WriteConfig.collection()
      • WriteConfig.connection()
      • WriteConfig.dataset_absolute_path()
      • WriteConfig.dataset_name()
      • WriteConfig.depot_details
      • WriteConfig.depot_name()
      • WriteConfig.depot_type()
      • WriteConfig.driver
      • WriteConfig.extra_options
      • WriteConfig.format_resolver()
      • WriteConfig.io_format
      • WriteConfig.is_stream
      • WriteConfig.mode
      • WriteConfig.spark_options
  • Module contents
• pyflare.sdk.core package
  • Submodules
  • pyflare.sdk.core.dataos_input module
    • DataOSInput
      • DataOSInput.process_inputs()
  • pyflare.sdk.core.dataos_output module
    • DataOSOutput
      • DataOSOutput.process_outputs()
  • pyflare.sdk.core.decorator module
  • pyflare.sdk.core.minerva_input module
    • MinervaInput
      • MinervaInput.process_inputs()
  • pyflare.sdk.core.session_builder module
    • SparkSessionBuilder
      • SparkSessionBuilder.add_reader_instance()
      • SparkSessionBuilder.add_writer_instance()
      • SparkSessionBuilder.api_token
      • SparkSessionBuilder.build_session()
      • SparkSessionBuilder.dataos_fqdn
      • SparkSessionBuilder.load_default_spark_conf()
      • SparkSessionBuilder.log_level
      • SparkSessionBuilder.logger
      • SparkSessionBuilder.parsed_inputs
      • SparkSessionBuilder.parsed_outputs
      • SparkSessionBuilder.spark
      • SparkSessionBuilder.spark_conf
      • SparkSessionBuilder.with_dataos_fqdn()
      • SparkSessionBuilder.with_depot()
      • SparkSessionBuilder.with_readers()
      • SparkSessionBuilder.with_spark_conf()
      • SparkSessionBuilder.with_user_apikey()
      • SparkSessionBuilder.with_writers()
    • g_dataos_token
    • g_inputs
    • g_outputs
    • load()
    • minerva_input()
    • refresh_global_data()
    • save()
    • spark
  • Module contents
• pyflare.sdk.depots package
  • Submodules
  • pyflare.sdk.depots.client module
    • DepotClientAPI
      • DepotClientAPI.get_depot_details()
  • Module contents
• pyflare.sdk.readers package
  • Submodules
  • pyflare.sdk.readers.bigquery_reader module
    • BigqueryInputReader
      • BigqueryInputReader.get_conf()
      • BigqueryInputReader.read()
      • BigqueryInputReader.read_stream()
  • pyflare.sdk.readers.delta_reader module
    • DeltaInputReader
      • DeltaInputReader.DELTA_CONF
      • DeltaInputReader.get_conf()
      • DeltaInputReader.read()
      • DeltaInputReader.read_stream()
  • pyflare.sdk.readers.elasticsearch_reader module
    • ElasticSearchInputReader
      • ElasticSearchInputReader.get_conf()
      • ElasticSearchInputReader.read()
      • ElasticSearchInputReader.read_stream()
  • pyflare.sdk.readers.fastbase_reader module
    • FastBaseInputReader
      • FastBaseInputReader.PULSAR_Options
      • FastBaseInputReader.get_conf()
      • FastBaseInputReader.read()
      • FastBaseInputReader.read_stream()
  • pyflare.sdk.readers.file_reader module
    • FileInputReader
      • FileInputReader.get_conf()
      • FileInputReader.read()
      • FileInputReader.read_stream()
  • pyflare.sdk.readers.iceberg_reader module
    • IcebergInputReader
      • IcebergInputReader.ICEBERG_CONF
      • IcebergInputReader.get_conf()
      • IcebergInputReader.read()
      • IcebergInputReader.read_stream()
  • pyflare.sdk.readers.jdbc_reader module
    • JDBCInputReader
      • JDBCInputReader.get_conf()
      • JDBCInputReader.read()
      • JDBCInputReader.read_stream()
  • pyflare.sdk.readers.minerva_reader module
    • MinervaInputReader
      • MinervaInputReader.get_conf()
      • MinervaInputReader.get_minevra_options()
      • MinervaInputReader.read()
      • MinervaInputReader.read_stream()
  • pyflare.sdk.readers.reader module
    • Reader
      • Reader.get_conf()
      • Reader.read()
      • Reader.read_config
      • Reader.read_stream()
      • Reader.spark
      • Reader.view_name
  • pyflare.sdk.readers.snowflake_reader module
    • SnowflakeInputReader
      • SnowflakeInputReader.SNOWFLAKE_READ_OPTIONS
      • SnowflakeInputReader.get_conf()
      • SnowflakeInputReader.read()
      • SnowflakeInputReader.read_stream()
      • SnowflakeInputReader.spark_options_conf()
  • Module contents
• pyflare.sdk.utils package
  • Submodules
  • pyflare.sdk.utils.generic_utils module
    • append_properties()
    • authorize_user()
    • decode_base64_string()
    • decorate_logger()
    • enhance_connection_url()
    • get_abfss_spark_conf()
    • get_dataset_path()
    • get_env_variable()
    • get_gcs_spark_conf()
    • get_jars_path()
    • get_s3_spark_conf()
    • get_secret_file_path()
    • get_secret_token()
    • resolve_dataos_address()
    • safe_assignment()
    • write_dict_to_file()
    • write_string_to_file()
  • pyflare.sdk.utils.pyflare_exceptions module
    • InvalidInputException
    • MissingElementException
    • MissingEnvironmentVariable
    • PyflareReadException
    • PyflareWriteException
  • pyflare.sdk.utils.pyflare_logger module
    • create_log4j_on_disk()
    • get_pyflare_logger()
    • log4j_log_level()
    • setup_pyflare_logger()
  • Module contents
• pyflare.sdk.writers package
  • Submodules
  • pyflare.sdk.writers.bigquery_writer module
    • BigqueryOutputWriter
      • BigqueryOutputWriter.get_conf()
      • BigqueryOutputWriter.write()
      • BigqueryOutputWriter.write_stream()
  • pyflare.sdk.writers.delta_writer module
    • DeltaOutputWriter
      • DeltaOutputWriter.DELTA_CONF
      • DeltaOutputWriter.get_conf()
      • DeltaOutputWriter.write()
      • DeltaOutputWriter.write_stream()
  • pyflare.sdk.writers.elasticsearch_writer module
    • ElasticSearchOutputWriter
      • ElasticSearchOutputWriter.get_conf()
      • ElasticSearchOutputWriter.write()
      • ElasticSearchOutputWriter.write_stream()
  • pyflare.sdk.writers.fastbase_writer module
    • FastBaseOutputWriter
      • FastBaseOutputWriter.PULSAR_Options
      • FastBaseOutputWriter.get_conf()
      • FastBaseOutputWriter.write()
      • FastBaseOutputWriter.write_stream()
  • pyflare.sdk.writers.file_writer module
    • FileOutputWriter
      • FileOutputWriter.get_conf()
      • FileOutputWriter.write()
      • FileOutputWriter.write_csv()
      • FileOutputWriter.write_json()
      • FileOutputWriter.write_parquet()
      • FileOutputWriter.write_stream()
  • pyflare.sdk.writers.iceberg_writer module
    • IcebergOutputWriter
      • IcebergOutputWriter.ICEBERG_CONF
      • IcebergOutputWriter.get_conf()
      • IcebergOutputWriter.write()
      • IcebergOutputWriter.write_stream()
  • pyflare.sdk.writers.jdbc_writer module
    • JDBCOutputWriter
      • JDBCOutputWriter.get_conf()
      • JDBCOutputWriter.write()
      • JDBCOutputWriter.write_stream()
  • pyflare.sdk.writers.snowflake_writer module
    • SnowflakeOutputWriter
      • SnowflakeOutputWriter.SNOWFLAKE_WRITE_OPTIONS
      • SnowflakeOutputWriter.get_conf()
      • SnowflakeOutputWriter.spark_options_conf()
      • SnowflakeOutputWriter.write()
      • SnowflakeOutputWriter.write_stream()
  • pyflare.sdk.writers.writer module
    • Writer
      • Writer.get_conf()
      • Writer.spark
      • Writer.view_name
      • Writer.write()
      • Writer.write_config
      • Writer.write_stream()
  • Module contents

Module contents

pyflare.sdk.load(name, format, driver=None, query=None, options=None)

pyflare.sdk.minerva_input(name, query, driver='io.trino.jdbc.TrinoDriver', options=None)

pyflare.sdk.save(name: str, dataframe, format: Optional[str] = None, mode='append', driver=None, options=None)