dataframe-arrow/org.jetbrains.kotlinx.dataframe.io

Package-level declarations

Types

internal object Allocator

class ~~ArrowFeather~~ : SupportedDataFrameFormat

Save dataFrame content in Apache Arrow format (can be written to File, ByteArray, OutputStream or raw Channel) with targetSchema. If dataFrame content does not match targetSchema, the behaviour is specified by mode, and mismatches would be sent to mismatchSubscriber

ArrowWriterImpl

internal class ArrowWriterImpl(val dataFrame: DataFrame<*>, val targetSchema: Schema, val mode: ArrowWriter.Mode, val mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage) : ArrowWriter

Save dataFrame content in Apache Arrow format (can be written to File, ByteArray, OutputStream or raw Channel) with targetSchema. If dataFrame content does not match with targetSchema, behaviour is specified by mode, mismatches would be sent to mismatchSubscriber

ConvertingException

class ConvertingException(val mismatchCase: ConvertingMismatch) : IllegalArgumentException

ConvertingMismatch

sealed class ConvertingMismatch(val column: String, val row: Int?, val cause: Exception?)

Detailed message about any mismatch when saving to Arrow format with a user-defined schema that does not match the actual data. Can be sent to a callback, written to a log, or wrapped in an exception

Properties

ARROW_PARQUET_DEFAULT_BATCH_SIZE

const val ARROW_PARQUET_DEFAULT_BATCH_SIZE: Long = 32768

Number of rows readParquet pulls from Arrow Dataset per scan batch, when no batchSize is given.

DEFAULT_INSTANT_UNIT

internal val DEFAULT_INSTANT_UNIT: TimeUnit

The precision an Instant column is written with when no target Schema is supplied.

deprecatedInstantType

internal val deprecatedInstantType: KType

kotlinx.datetime.Instant is superseded by kotlin.time.Instant but still resolvable, so columns holding it are mapped too. Kept in a val to keep the deprecation suppression off the whole when. See issue #1350.

ignoreMismatchMessage

val ignoreMismatchMessage: (ConvertingMismatch) -> Unit

logMismatchMessage

val logMismatchMessage: (ConvertingMismatch) -> Unit

NANOS_PER_SECOND

internal const val NANOS_PER_SECOND: Long

nullableNothingType

internal val nullableNothingType: KType

perSecond

internal val TimeUnit.perSecond: Long

How many of these units fit into one second.

writeMismatchMessage

val writeMismatchMessage: (ConvertingMismatch) -> Unit

Functions

arrowWriter

fun AnyFrame.arrowWriter(): ArrowWriter

Create ArrowWriter for this DataFrame with target schema matching actual data

fun AnyFrame.arrowWriter(targetSchema: Schema, mode: ArrowWriter.Mode = ArrowWriter.Mode.STRICT, mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage): ArrowWriter

Create ArrowWriter for this DataFrame with explicit targetSchema. If DataFrame does not match with targetSchema, behaviour is specified by mode, mismatches would be sent to mismatchSubscriber

concatKeepingSchema

internal fun <T> Iterable<DataFrame<T>>.concatKeepingSchema(): DataFrame<T>

same as Iterable>.concat() without internal type guessing (all batches should have the same schema)

nothingType

internal fun nothingType(nullable: Boolean): KType

readArrow

fun DataFrame.Companion.readArrow(reader: ArrowReader, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow any format data from existing reader

readArrowDatasetImpl

internal fun DataFrame.Companion.readArrowDatasetImpl(fileUris: Array<String>, fileFormat: FileFormat, nullability: NullabilityOptions = NullabilityOptions.Infer, batchSize: Long = ARROW_PARQUET_DEFAULT_BATCH_SIZE): AnyFrame

Read Arrow Dataset from fileUris

readArrowFeather

fun DataFrame.Companion.readArrowFeather(file: File, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow random access format data from existing file

fun DataFrame.Companion.readArrowFeather(stream: InputStream, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow random access format data from existing stream

fun DataFrame.Companion.readArrowFeather(url: URL, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow random access format data from existing url

fun DataFrame.Companion.readArrowFeather(path: Path, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow random access format data from an existing file on the given path.

fun DataFrame.Companion.readArrowFeather(byteArray: ByteArray, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow random access format data from existing byteArray

fun DataFrame.Companion.readArrowFeather(path: String, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow random access format data from existing path

fun DataFrame.Companion.readArrowFeather(channel: SeekableByteChannel, allocator: RootAllocator = Allocator.ROOT, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow random access format data from existing channel

readArrowFeatherImpl

internal fun DataFrame.Companion.readArrowFeatherImpl(channel: SeekableByteChannel, allocator: RootAllocator = Allocator.ROOT, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow random access format data from existing channel

readArrowImpl

internal fun DataFrame.Companion.readArrowImpl(reader: ArrowReader, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow any format data from existing reader

readArrowIPC

fun DataFrame.Companion.readArrowIPC(file: File, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow interprocess streaming format data from existing file

fun DataFrame.Companion.readArrowIPC(stream: InputStream, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow interprocess streaming format data from existing stream

fun DataFrame.Companion.readArrowIPC(url: URL, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow interprocess streaming format data from existing url

fun DataFrame.Companion.readArrowIPC(path: Path, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow interprocess streaming format data from existing file on the given path.

fun DataFrame.Companion.readArrowIPC(byteArray: ByteArray, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow interprocess streaming format data from existing byteArray

fun DataFrame.Companion.readArrowIPC(path: String, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow interprocess streaming format data from the file or URL at path.

fun DataFrame.Companion.readArrowIPC(channel: ReadableByteChannel, allocator: RootAllocator = Allocator.ROOT, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow interprocess streaming format data from existing channel

readArrowIPCImpl

internal fun DataFrame.Companion.readArrowIPCImpl(channel: ReadableByteChannel, allocator: RootAllocator = Allocator.ROOT, nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow interprocess streaming format data from existing channel

readParquet

fun DataFrame.Companion.readParquet(vararg files: File, nullability: NullabilityOptions = NullabilityOptions.Infer, batchSize: Long = ARROW_PARQUET_DEFAULT_BATCH_SIZE): AnyFrame

Read Parquet data from existing files by using Arrow Dataset

fun DataFrame.Companion.readParquet(vararg urls: URL, nullability: NullabilityOptions = NullabilityOptions.Infer, batchSize: Long = ARROW_PARQUET_DEFAULT_BATCH_SIZE): AnyFrame

Read Parquet data from existing urls by using Arrow Dataset

fun DataFrame.Companion.readParquet(vararg paths: Path, nullability: NullabilityOptions = NullabilityOptions.Infer, batchSize: Long = ARROW_PARQUET_DEFAULT_BATCH_SIZE): AnyFrame

Read Parquet data from existing paths by using Arrow Dataset

fun DataFrame.Companion.readParquet(vararg strUrls: String, nullability: NullabilityOptions = NullabilityOptions.Infer, batchSize: Long = ARROW_PARQUET_DEFAULT_BATCH_SIZE): AnyFrame

Read Parquet data from existing strUrls by using Arrow Dataset

saveArrowFeatherToByteArray

fun AnyFrame.saveArrowFeatherToByteArray(): ByteArray

Save data to Arrow random access format, write to new ByteArray

saveArrowIPCToByteArray

fun AnyFrame.saveArrowIPCToByteArray(): ByteArray

Save data to Arrow interprocess streaming format, write to new ByteArray

toArrowField

fun DataColumn<*>.toArrowField(mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage): Field

Create Arrow Field (note: this is part of Schema, does not contain data itself) that has the same name, type and nullability as this

internal fun KType.toArrowField(name: String, mismatchSubscriber: (ConvertingMismatch) -> Unit): Field

internal fun ColumnSchema.toArrowField(name: String, mismatchSubscriber: (ConvertingMismatch) -> Unit): Field

toArrowSchema

fun List<DataColumn<*>>.toArrowSchema(mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage): Schema

Create Arrow Schema matching this actual data. Columns with not supported types will be interpreted as String

toDataFrame

fun ArrowReader.toDataFrame(nullability: NullabilityOptions = NullabilityOptions.Infer): AnyFrame

Read Arrow any format data from existing ArrowReader

writeArrowFeather

fun AnyFrame.writeArrowFeather(file: File)

Save data to Arrow random access format, write to new or existing file. If file exists, it would be recreated.

fun AnyFrame.writeArrowFeather(stream: OutputStream)

Save data to Arrow random access format, write to opened stream.

fun AnyFrame.writeArrowFeather(channel: WritableByteChannel)

Save data to Arrow random access format, write to opened channel.

fun AnyFrame.writeArrowFeather(path: Path)

Save data to Arrow random access format, write to new or existing file on the given path. If file exists, it would be recreated.

writeArrowIPC

fun AnyFrame.writeArrowIPC(stream: OutputStream)

Save data to Arrow interprocess streaming format, write to opened stream.

fun AnyFrame.writeArrowIPC(channel: WritableByteChannel)

Save data to Arrow interprocess streaming format, write to opened channel.

fun AnyFrame.writeArrowIPC(file: File, append: Boolean = true)

Save data to Arrow interprocess streaming format, write to new or existing file. If file exists, it can be recreated or expanded.

fun AnyFrame.writeArrowIPC(path: Path, append: Boolean = true)

Save data to Arrow interprocess streaming format, write to new or existing file on the given path. If file exists, it can be recreated or expanded.