case class FilesInput(name: String, paths: Seq[String], warehousePath: Option[String], catalogName: Option[String], schemaName: Option[String], tableName: Option[String], metastoreUris: Option[String], options: Option[Map[String, String]], format: Option[String], isMetric: Option[Boolean], icebergCatalogType: Option[String], incremental: Option[Incremental], schema: Option[Schema]) extends FileInputBase with Product with Serializable
Represents a file-based input configuration.
- name
The name of the file input.
- paths
The sequence of file paths for the input.
- warehousePath
An optional warehouse path for the input.
- catalogName
An optional catalog name for the input.
- schemaName
An optional schema name for the input.
- tableName
An optional table name for the input.
- metastoreUris
An optional string representing metastore URIs for the input.
- options
An optional map of options for the input.
- format
An optional string representing the format of the input.
- isMetric
An optional flag indicating if the input is for metrics.
- icebergCatalogType
An optional string representing the catalog type for Iceberg format.
- incremental
An optional Incremental configuration for the input.
- schema
An optional Schema for the input.
- Alphabetic
- By Inheritance
- FilesInput
- Serializable
- Serializable
- Product
- Equals
- FileInputBase
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
-
new
FilesInput(name: String, paths: Seq[String], warehousePath: Option[String], catalogName: Option[String], schemaName: Option[String], tableName: Option[String], metastoreUris: Option[String], options: Option[Map[String, String]], format: Option[String], isMetric: Option[Boolean], icebergCatalogType: Option[String], incremental: Option[Incremental], schema: Option[Schema])
- name
The name of the file input.
- paths
The sequence of file paths for the input.
- warehousePath
An optional warehouse path for the input.
- catalogName
An optional catalog name for the input.
- schemaName
An optional schema name for the input.
- tableName
An optional table name for the input.
- metastoreUris
An optional string representing metastore URIs for the input.
- options
An optional map of options for the input.
- format
An optional string representing the format of the input.
- isMetric
An optional flag indicating if the input is for metrics.
- icebergCatalogType
An optional string representing the catalog type for Iceberg format.
- incremental
An optional Incremental configuration for the input.
- schema
An optional Schema for the input.
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
- val catalogName: Option[String]
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
- val format: Option[String]
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getFormat(format: Option[String], path: String): String
Determines the file format based on the specified format option or file path.
Determines the file format based on the specified format option or file path.
- format
The optional format specified by the user.
- path
The file path.
- returns
The determined file format.
- Definition Classes
- FileInputBase
-
def
getOptions(readFormat: String, options: Option[Map[String, String]]): Option[Map[String, String]]
Retrieves the options for reading files based on the file format.
Retrieves the options for reading files based on the file format.
- readFormat
The file format.
- options
The optional user-provided options.
- returns
The options for reading files.
- Definition Classes
- FileInputBase
-
def
getSchemaStruct(schema: Option[Schema], sparkSession: SparkSession): Option[StructType]
Retrieves the schema structure based on the provided schema path, schema string, and schema type.
Retrieves the schema structure based on the provided schema path, schema string, and schema type.
- schema
Schema info
- sparkSession
The Spark session.
- returns
The schema structure as a
StructType.
- Definition Classes
- FileInputBase
-
def
getTable: String
Generates the fully qualified table name in the format:
catalogName.schemaName.tableName.Generates the fully qualified table name in the format:
catalogName.schemaName.tableName.- returns
The fully qualified table name.
- val icebergCatalogType: Option[String]
- val incremental: Option[Incremental]
-
def
isHiveCatalog(format: String): Boolean
Checks if the specified file format corresponds to a Hive catalog.
Checks if the specified file format corresponds to a Hive catalog.
- format
The file format.
- returns
trueif the format corresponds to a Hive catalog,falseotherwise.
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- val isMetric: Option[Boolean]
- val metastoreUris: Option[String]
- val name: String
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- val options: Option[Map[String, String]]
- val paths: Seq[String]
-
def
read(sparkSession: SparkSession): DataFrame
Reads the files using the specified SparkSession and returns the resulting DataFrame.
Reads the files using the specified SparkSession and returns the resulting DataFrame.
- sparkSession
The SparkSession.
- returns
The DataFrame containing the read data.
- val schema: Option[Schema]
- val schemaName: Option[String]
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
- val tableName: Option[String]
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
- val warehousePath: Option[String]