Index

A

add_reader_instance() (pyflare.sdk.core.session_builder.SparkSessionBuilder method) add_writer_instance() (pyflare.sdk.core.session_builder.SparkSessionBuilder method)

api_token (pyflare.sdk.core.session_builder.SparkSessionBuilder attribute) append_properties() (in module pyflare.sdk.utils.generic_utils) authorize_user() (in module pyflare.sdk.utils.generic_utils)

B

BigqueryInputReader (class in pyflare.sdk.readers.bigquery_reader)

BigqueryOutputWriter (class in pyflare.sdk.writers.bigquery_writer) build_session() (pyflare.sdk.core.session_builder.SparkSessionBuilder method)

C

collection() (pyflare.sdk.config.read_config.ReadConfig method) (pyflare.sdk.config.write_config.WriteConfig method)

connection() (pyflare.sdk.config.read_config.ReadConfig method) (pyflare.sdk.config.write_config.WriteConfig method) create_log4j_on_disk() (in module pyflare.sdk.utils.pyflare_logger)

D

dataos_fqdn (pyflare.sdk.core.session_builder.SparkSessionBuilder attribute) DataOSInput (class in pyflare.sdk.core.dataos_input) DataOSOutput (class in pyflare.sdk.core.dataos_output) dataset_absolute_path() (pyflare.sdk.config.read_config.ReadConfig method) (pyflare.sdk.config.write_config.WriteConfig method) dataset_name() (pyflare.sdk.config.read_config.ReadConfig method) (pyflare.sdk.config.write_config.WriteConfig method) decode_base64_string() (in module pyflare.sdk.utils.generic_utils) decorate_logger() (in module pyflare.sdk.utils.generic_utils) DELTA_CONF (pyflare.sdk.readers.delta_reader.DeltaInputReader attribute) (pyflare.sdk.writers.delta_writer.DeltaOutputWriter attribute)

DeltaInputReader (class in pyflare.sdk.readers.delta_reader) DeltaOutputWriter (class in pyflare.sdk.writers.delta_writer) depot_details (pyflare.sdk.config.read_config.ReadConfig property) (pyflare.sdk.config.write_config.WriteConfig property) depot_name() (pyflare.sdk.config.read_config.ReadConfig method) (pyflare.sdk.config.write_config.WriteConfig method) depot_type() (pyflare.sdk.config.read_config.ReadConfig method) (pyflare.sdk.config.write_config.WriteConfig method) DepotClientAPI (class in pyflare.sdk.depots.client) driver (pyflare.sdk.config.read_config.ReadConfig property) (pyflare.sdk.config.write_config.WriteConfig property)

E

ElasticSearchInputReader (class in pyflare.sdk.readers.elasticsearch_reader) ElasticSearchOutputWriter (class in pyflare.sdk.writers.elasticsearch_writer)

enhance_connection_url() (in module pyflare.sdk.utils.generic_utils) extra_options (pyflare.sdk.config.read_config.ReadConfig property) (pyflare.sdk.config.write_config.WriteConfig property)

F

FastBaseInputReader (class in pyflare.sdk.readers.fastbase_reader) FastBaseOutputWriter (class in pyflare.sdk.writers.fastbase_writer) FileInputReader (class in pyflare.sdk.readers.file_reader)

FileOutputWriter (class in pyflare.sdk.writers.file_writer) format_resolver() (pyflare.sdk.config.read_config.ReadConfig method) (pyflare.sdk.config.write_config.WriteConfig method)

G

g_dataos_token (in module pyflare.sdk.core.session_builder) g_inputs (in module pyflare.sdk.core.session_builder) g_outputs (in module pyflare.sdk.core.session_builder) get_abfss_spark_conf() (in module pyflare.sdk.utils.generic_utils) get_conf() (pyflare.sdk.readers.bigquery_reader.BigqueryInputReader method) (pyflare.sdk.readers.delta_reader.DeltaInputReader method) (pyflare.sdk.readers.elasticsearch_reader.ElasticSearchInputReader method) (pyflare.sdk.readers.fastbase_reader.FastBaseInputReader method) (pyflare.sdk.readers.file_reader.FileInputReader method) (pyflare.sdk.readers.iceberg_reader.IcebergInputReader method) (pyflare.sdk.readers.jdbc_reader.JDBCInputReader method) (pyflare.sdk.readers.minerva_reader.MinervaInputReader method) (pyflare.sdk.readers.reader.Reader method) (pyflare.sdk.readers.snowflake_reader.SnowflakeInputReader method) (pyflare.sdk.writers.bigquery_writer.BigqueryOutputWriter method) (pyflare.sdk.writers.delta_writer.DeltaOutputWriter method) (pyflare.sdk.writers.elasticsearch_writer.ElasticSearchOutputWriter method) (pyflare.sdk.writers.fastbase_writer.FastBaseOutputWriter method) (pyflare.sdk.writers.file_writer.FileOutputWriter method) (pyflare.sdk.writers.iceberg_writer.IcebergOutputWriter method) (pyflare.sdk.writers.jdbc_writer.JDBCOutputWriter method) (pyflare.sdk.writers.snowflake_writer.SnowflakeOutputWriter method) (pyflare.sdk.writers.writer.Writer method)

get_dataset_path() (in module pyflare.sdk.utils.generic_utils) get_depot_details() (pyflare.sdk.depots.client.DepotClientAPI method) get_env_variable() (in module pyflare.sdk.utils.generic_utils) get_gcs_spark_conf() (in module pyflare.sdk.utils.generic_utils) get_jars_path() (in module pyflare.sdk.utils.generic_utils) get_log4j_properties_path() (in module pyflare.sdk.config.constants) get_minevra_options() (pyflare.sdk.readers.minerva_reader.MinervaInputReader method) get_pyflare_logger() (in module pyflare.sdk.utils.pyflare_logger) get_s3_spark_conf() (in module pyflare.sdk.utils.generic_utils) get_secret_file_path() (in module pyflare.sdk.utils.generic_utils) get_secret_token() (in module pyflare.sdk.utils.generic_utils) get_spark_app_name() (in module pyflare.sdk.config.constants) get_spark_conf() (in module pyflare.secrets.abfss) (in module pyflare.secrets.s3) get_value_or_throw() (in module pyflare.secrets.abfss) (in module pyflare.secrets.gcs) (in module pyflare.secrets.s3)

H

has_spark_secrets() (in module pyflare.secrets.gcs) (in module pyflare.secrets.s3)

I

ICEBERG_CONF (pyflare.sdk.readers.iceberg_reader.IcebergInputReader attribute) (pyflare.sdk.writers.iceberg_writer.IcebergOutputWriter attribute) IcebergInputReader (class in pyflare.sdk.readers.iceberg_reader) IcebergOutputWriter (class in pyflare.sdk.writers.iceberg_writer) InvalidInputException

io_format (pyflare.sdk.config.read_config.ReadConfig property) (pyflare.sdk.config.write_config.WriteConfig property) is_secrets_loaded() (in module pyflare.secrets) is_stream (pyflare.sdk.config.read_config.ReadConfig property) (pyflare.sdk.config.write_config.WriteConfig property)

J

JDBCInputReader (class in pyflare.sdk.readers.jdbc_reader)

JDBCOutputWriter (class in pyflare.sdk.writers.jdbc_writer)

L

load() (in module pyflare) (in module pyflare.sdk) (in module pyflare.sdk.core.session_builder)

load_default_spark_conf() (pyflare.sdk.core.session_builder.SparkSessionBuilder method) log4j_log_level() (in module pyflare.sdk.utils.pyflare_logger) log_level (pyflare.sdk.core.session_builder.SparkSessionBuilder attribute) logger (pyflare.sdk.core.session_builder.SparkSessionBuilder attribute)

M

minerva_input() (in module pyflare) (in module pyflare.sdk) (in module pyflare.sdk.core.session_builder) MinervaInput (class in pyflare.sdk.core.minerva_input) MinervaInputReader (class in pyflare.sdk.readers.minerva_reader) MissingElementException MissingEnvironmentVariable mode (pyflare.sdk.config.write_config.WriteConfig property) module pyflare pyflare.jars pyflare.sdk pyflare.sdk.config pyflare.sdk.config.constants pyflare.sdk.config.read_config pyflare.sdk.config.write_config pyflare.sdk.core pyflare.sdk.core.dataos_input pyflare.sdk.core.dataos_output pyflare.sdk.core.decorator pyflare.sdk.core.minerva_input pyflare.sdk.core.session_builder pyflare.sdk.depots pyflare.sdk.depots.client pyflare.sdk.readers pyflare.sdk.readers.bigquery_reader pyflare.sdk.readers.delta_reader pyflare.sdk.readers.elasticsearch_reader pyflare.sdk.readers.fastbase_reader pyflare.sdk.readers.file_reader pyflare.sdk.readers.iceberg_reader pyflare.sdk.readers.jdbc_reader pyflare.sdk.readers.minerva_reader pyflare.sdk.readers.reader pyflare.sdk.readers.snowflake_reader pyflare.sdk.utils pyflare.sdk.utils.generic_utils pyflare.sdk.utils.pyflare_exceptions pyflare.sdk.utils.pyflare_logger pyflare.sdk.writers pyflare.sdk.writers.bigquery_writer pyflare.sdk.writers.delta_writer pyflare.sdk.writers.elasticsearch_writer pyflare.sdk.writers.fastbase_writer pyflare.sdk.writers.file_writer pyflare.sdk.writers.iceberg_writer pyflare.sdk.writers.jdbc_writer pyflare.sdk.writers.snowflake_writer pyflare.sdk.writers.writer pyflare.secrets pyflare.secrets.abfss pyflare.secrets.gcs pyflare.secrets.s3

P

parsed_inputs (pyflare.sdk.core.session_builder.SparkSessionBuilder attribute) parsed_outputs (pyflare.sdk.core.session_builder.SparkSessionBuilder attribute) process_inputs() (pyflare.sdk.core.dataos_input.DataOSInput method) (pyflare.sdk.core.minerva_input.MinervaInput method) process_outputs() (pyflare.sdk.core.dataos_output.DataOSOutput method) PULSAR_Options (pyflare.sdk.readers.fastbase_reader.FastBaseInputReader attribute) (pyflare.sdk.writers.fastbase_writer.FastBaseOutputWriter attribute) pyflare module pyflare.jars module pyflare.sdk module pyflare.sdk.config module pyflare.sdk.config.constants module pyflare.sdk.config.read_config module pyflare.sdk.config.write_config module pyflare.sdk.core module pyflare.sdk.core.dataos_input module pyflare.sdk.core.dataos_output module pyflare.sdk.core.decorator module pyflare.sdk.core.minerva_input module pyflare.sdk.core.session_builder module pyflare.sdk.depots module pyflare.sdk.depots.client module pyflare.sdk.readers module pyflare.sdk.readers.bigquery_reader module pyflare.sdk.readers.delta_reader module pyflare.sdk.readers.elasticsearch_reader module pyflare.sdk.readers.fastbase_reader module pyflare.sdk.readers.file_reader module

pyflare.sdk.readers.iceberg_reader module pyflare.sdk.readers.jdbc_reader module pyflare.sdk.readers.minerva_reader module pyflare.sdk.readers.reader module pyflare.sdk.readers.snowflake_reader module pyflare.sdk.utils module pyflare.sdk.utils.generic_utils module pyflare.sdk.utils.pyflare_exceptions module pyflare.sdk.utils.pyflare_logger module pyflare.sdk.writers module pyflare.sdk.writers.bigquery_writer module pyflare.sdk.writers.delta_writer module pyflare.sdk.writers.elasticsearch_writer module pyflare.sdk.writers.fastbase_writer module pyflare.sdk.writers.file_writer module pyflare.sdk.writers.iceberg_writer module pyflare.sdk.writers.jdbc_writer module pyflare.sdk.writers.snowflake_writer module pyflare.sdk.writers.writer module pyflare.secrets module pyflare.secrets.abfss module pyflare.secrets.gcs module pyflare.secrets.s3 module PyflareReadException PyflareWriteException

Q

query (pyflare.sdk.config.read_config.ReadConfig property)

R

read() (pyflare.sdk.readers.bigquery_reader.BigqueryInputReader method) (pyflare.sdk.readers.delta_reader.DeltaInputReader method) (pyflare.sdk.readers.elasticsearch_reader.ElasticSearchInputReader method) (pyflare.sdk.readers.fastbase_reader.FastBaseInputReader method) (pyflare.sdk.readers.file_reader.FileInputReader method) (pyflare.sdk.readers.iceberg_reader.IcebergInputReader method) (pyflare.sdk.readers.jdbc_reader.JDBCInputReader method) (pyflare.sdk.readers.minerva_reader.MinervaInputReader method) (pyflare.sdk.readers.reader.Reader method) (pyflare.sdk.readers.snowflake_reader.SnowflakeInputReader method) read_config (pyflare.sdk.readers.reader.Reader property) read_stream() (pyflare.sdk.readers.bigquery_reader.BigqueryInputReader method) (pyflare.sdk.readers.delta_reader.DeltaInputReader method) (pyflare.sdk.readers.elasticsearch_reader.ElasticSearchInputReader method) (pyflare.sdk.readers.fastbase_reader.FastBaseInputReader method) (pyflare.sdk.readers.file_reader.FileInputReader method) (pyflare.sdk.readers.iceberg_reader.IcebergInputReader method) (pyflare.sdk.readers.jdbc_reader.JDBCInputReader method) (pyflare.sdk.readers.minerva_reader.MinervaInputReader method) (pyflare.sdk.readers.reader.Reader method) (pyflare.sdk.readers.snowflake_reader.SnowflakeInputReader method)

ReadConfig (class in pyflare.sdk.config.read_config) Reader (class in pyflare.sdk.readers.reader) refresh_global_data() (in module pyflare.sdk.core.session_builder) resolve_dataos_address() (in module pyflare.sdk.utils.generic_utils)

S

safe_assignment() (in module pyflare.sdk.utils.generic_utils) save() (in module pyflare) (in module pyflare.sdk) (in module pyflare.sdk.core.session_builder) setup_pyflare_logger() (in module pyflare.sdk.utils.pyflare_logger) SNOWFLAKE_READ_OPTIONS (pyflare.sdk.readers.snowflake_reader.SnowflakeInputReader attribute) SNOWFLAKE_WRITE_OPTIONS (pyflare.sdk.writers.snowflake_writer.SnowflakeOutputWriter attribute) SnowflakeInputReader (class in pyflare.sdk.readers.snowflake_reader) SnowflakeOutputWriter (class in pyflare.sdk.writers.snowflake_writer)

spark (in module pyflare.sdk.core.session_builder) (pyflare.sdk.core.session_builder.SparkSessionBuilder attribute) (pyflare.sdk.readers.reader.Reader property) (pyflare.sdk.writers.writer.Writer property) spark_conf (pyflare.sdk.core.session_builder.SparkSessionBuilder attribute) spark_options (pyflare.sdk.config.read_config.ReadConfig property) (pyflare.sdk.config.write_config.WriteConfig property) spark_options_conf() (pyflare.sdk.readers.snowflake_reader.SnowflakeInputReader method) (pyflare.sdk.writers.snowflake_writer.SnowflakeOutputWriter method) SparkSessionBuilder (class in pyflare.sdk.core.session_builder)

V

view_name (pyflare.sdk.readers.reader.Reader property) (pyflare.sdk.writers.writer.Writer property)

W

with_dataos_fqdn() (pyflare.sdk.core.session_builder.SparkSessionBuilder method) with_depot() (pyflare.sdk.core.session_builder.SparkSessionBuilder method) with_readers() (pyflare.sdk.core.session_builder.SparkSessionBuilder method) with_spark_conf() (pyflare.sdk.core.session_builder.SparkSessionBuilder method) with_user_apikey() (pyflare.sdk.core.session_builder.SparkSessionBuilder method) with_writers() (pyflare.sdk.core.session_builder.SparkSessionBuilder method) write() (pyflare.sdk.writers.bigquery_writer.BigqueryOutputWriter method) (pyflare.sdk.writers.delta_writer.DeltaOutputWriter method) (pyflare.sdk.writers.elasticsearch_writer.ElasticSearchOutputWriter method) (pyflare.sdk.writers.fastbase_writer.FastBaseOutputWriter method) (pyflare.sdk.writers.file_writer.FileOutputWriter method) (pyflare.sdk.writers.iceberg_writer.IcebergOutputWriter method) (pyflare.sdk.writers.jdbc_writer.JDBCOutputWriter method) (pyflare.sdk.writers.snowflake_writer.SnowflakeOutputWriter method) (pyflare.sdk.writers.writer.Writer method) write_config (pyflare.sdk.writers.writer.Writer property)

write_csv() (pyflare.sdk.writers.file_writer.FileOutputWriter method) write_dict_to_file() (in module pyflare.sdk.utils.generic_utils) write_json() (pyflare.sdk.writers.file_writer.FileOutputWriter method) write_parquet() (pyflare.sdk.writers.file_writer.FileOutputWriter method) write_stream() (pyflare.sdk.writers.bigquery_writer.BigqueryOutputWriter method) (pyflare.sdk.writers.delta_writer.DeltaOutputWriter method) (pyflare.sdk.writers.elasticsearch_writer.ElasticSearchOutputWriter method) (pyflare.sdk.writers.fastbase_writer.FastBaseOutputWriter method) (pyflare.sdk.writers.file_writer.FileOutputWriter method) (pyflare.sdk.writers.iceberg_writer.IcebergOutputWriter method) (pyflare.sdk.writers.jdbc_writer.JDBCOutputWriter method) (pyflare.sdk.writers.snowflake_writer.SnowflakeOutputWriter method) (pyflare.sdk.writers.writer.Writer method) write_string_to_file() (in module pyflare.sdk.utils.generic_utils) WriteConfig (class in pyflare.sdk.config.write_config) Writer (class in pyflare.sdk.writers.writer)