pyflare.sdk.writers package

Submodules

pyflare.sdk.writers.bigquery_writer module

class pyflare.sdk.writers.bigquery_writer.BigqueryOutputWriter(write_config: WriteConfig)[source]

Bases: Writer

get_conf()[source]
write(df)[source]
write_stream()[source]

pyflare.sdk.writers.delta_writer module

class pyflare.sdk.writers.delta_writer.DeltaOutputWriter(write_config: WriteConfig)[source]

Bases: FileOutputWriter

DELTA_CONF = '[\n            ("spark.sql.catalog.spark_catalog","org.apache.spark.sql.delta.catalog.DeltaCatalog"),\n            ("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")\n        ]'
get_conf()[source]
write(df)[source]
write_stream()[source]

pyflare.sdk.writers.elasticsearch_writer module

class pyflare.sdk.writers.elasticsearch_writer.ElasticSearchOutputWriter(write_config: WriteConfig)[source]

Bases: Writer

get_conf()[source]
write(df)[source]
write_stream()[source]

pyflare.sdk.writers.fastbase_writer module

class pyflare.sdk.writers.fastbase_writer.FastBaseOutputWriter(write_config: WriteConfig)[source]

Bases: FileOutputWriter

PULSAR_Options = '[\n            ("service.url", "{serviceUrl}"),\n            ("admin.url", "{adminUrl}"),\n            ("pulsar.admin.authPluginClassName", "org.apache.pulsar.client.impl.auth.AuthenticationToken"),\n            ("pulsar.admin.authParams","token:{Apikey}"),\n            ("pulsar.client.authPluginClassName","org.apache.pulsar.client.impl.auth.AuthenticationToken"),\n            ("pulsar.client.authParams","token:{Apikey}"),\n            ("topic","persistent://public/default/{dataset}")\n        ]'
get_conf()[source]
write(df)[source]
write_stream()[source]

pyflare.sdk.writers.file_writer module

class pyflare.sdk.writers.file_writer.FileOutputWriter(write_config: WriteConfig)[source]

Bases: Writer

get_conf()[source]
write(df)[source]
write_csv()[source]
write_json()[source]
write_parquet()[source]
write_stream()[source]

pyflare.sdk.writers.iceberg_writer module

class pyflare.sdk.writers.iceberg_writer.IcebergOutputWriter(write_config: WriteConfig)[source]

Bases: FileOutputWriter

A class for writing data to the iceberg format using PySpark.

cleanup_temp_spark_conf()[source]

Cleans up the temporary Spark configuration for iceberg.

Returns:

None

get_conf()[source]

Returns spark configuration required for iceberg as per underlying filesystem.

Returns:

Spark configuration required for iceberg as per underlying filesystem.

Return type:

list of tuples

temp_spark_conf()[source]

Generates a temporary Spark configuration for iceberg.

Returns:

The temporary Spark configuration.

Return type:

list of tuples

write(df: DataFrame)[source]

Writes the given DataFrame to the iceberg format. If the “merge” option is specified in the write_config’s extra_options, the DataFrame is merged into an existing table using a temporary view. Otherwise, the DataFrame is written to the iceberg format with the given options.

Parameters:

df (DataFrame) – The DataFrame to write.

Returns:

None

write_stream()[source]

Writes the given DataFrame to the iceberg format in streaming mode.

Returns:

None

pyflare.sdk.writers.jdbc_writer module

class pyflare.sdk.writers.jdbc_writer.JDBCOutputWriter(write_config: WriteConfig)[source]

Bases: Writer

get_conf()[source]
write(df)[source]
write_stream()[source]

pyflare.sdk.writers.snowflake_writer module

class pyflare.sdk.writers.snowflake_writer.SnowflakeOutputWriter(write_config: WriteConfig)[source]

Bases: Writer

SNOWFLAKE_WRITE_OPTIONS = '{{\n                "sfURL": "{connection_url}",\n                "sfUser": "{connection_user}",\n                "sfPassword": "{connection_password}",\n                "sfDatabase": "{connection_database}",\n                "sfSchema": "{collection}",\n                "sfWarehouse": "{connection_warehouse}",\n                "dbtable": "{dataset}"\n            }}'
get_conf()[source]
spark_options_conf()[source]
write(df)[source]
write_stream()[source]

pyflare.sdk.writers.writer module

class pyflare.sdk.writers.writer.Writer(write_config: WriteConfig)[source]

Bases: object

abstract get_conf()[source]
property spark: SparkSession
property view_name
abstract write(df)[source]
property write_config: WriteConfig
abstract write_stream()[source]

Module contents