diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6eba671..7e935bc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,8 +51,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml index 06ff608..99132a9 100644 --- a/.github/workflows/erlang.yml +++ b/.github/workflows/erlang.yml @@ -43,8 +43,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -81,8 +81,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -122,8 +122,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/rebar.config b/rebar.config index 5a757fa..46065cd 100644 --- a/rebar.config +++ b/rebar.config @@ -29,7 +29,7 @@ ]}. {plugins, [rebar3_cargo]}. -{project_plugins, [erlfmt, {rebar3_ex_doc, "0.2.18"}]}. +{project_plugins, [erlfmt, {rebar3_ex_doc, "0.2.30"}]}. % Let erlfmt write files on format as opposed to not doing anything {erlfmt, [write]}. diff --git a/src/arrow_array.erl b/src/arrow_array.erl index 9c2045b..56a89a4 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -15,87 +15,54 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record, a behaviour, and functions to work with Apache Arrow -%% Arrays. -%% -%% This module as mentioned, provides conveniences for working with Apache Arrow -%% Arrays[1] of different Layouts[2]. Firstly, it provides a record to represent -%% all the Layout. Secondly, it provides a behaviour for different layouts to -%% adhere to common functionality. Lastly, it provides functions to work with -%% Arrays in a common manner. -%% -%% == The Structure of an Array == -%% -%% `arrow''s implementation of an Array has the following fields in its -%% record definition: -%% -%%
    -%%
  1. -%% `layout', of type {@link atom()}, which represents the Layout of the -%% Array. -%%
    2. -%%
  2. -%% `type', of type {@link arrow_type:arrow_type()}, which represents -%% the Logical Type[3] of the Array. -%%
    3. -%%
  3. `len', of type {@link pos_integer()}, which represents the Array's Length[4].
    4. -%%
  4. -%% `element_len', of type {@link pos_integer()} or `undefined', which -%% represents the Length of each element in an Array. Currently it only has -%% an integer value in the Fixed-Size List Layout[5] -%%
    5. -%%
  5. -%% `null_count', of type {@link non_neg_integer()}, which represents the -%% Array's Null Count[4], or the number of undefined values in the Array. -%%
    6. -%%
  6. -%% `validity_bitmap', which is a buffer (`arrow_buffer') or the atom -%% `undefined', which represents the Array's Validity Bitmap[6]. -%%
    7. -%%
  7. -%% `offsets', which is a buffer (`arrow_buffer') which represents the -%% Offsets[7], or the start position of each slot in the data buffer of an -%% Array. -%%
    8. -%%
  8. -%% `data', which is a buffer (`arrow_buffer'), which represents the -%% Array's Value Buffer, whose layout differs based on the Array Layout. -%%
    9. -%%
-%% -%% Certain fields are not required for certain layouts. For example, for the -%% Fixed-Sized Primitive Layout, the offsets field is not required, in which -%% case it is assigned as `undefined'. Similarly, the `validity_bitmap' is not -%% required if there are no null values, in which case it is also assigned as -%% `undefined'. -%% -%% == The Behaviour of an Array == -%% -%% As of right now, a layout needs to implement the `c:from_erlang/2' callback. This is -%% then used in the `from_erlang/3' function to create new arrays. -%% -%% == Functions for working with Arrays == -%% -%% As of right now, only functions to access the various fields, to create -%% new arrays, and to serialize arrays to arrow exist. -%% -%% == References == -%% -%% [1]: [https://arrow.apache.org/docs/format/Glossary.html#term-array] -%% -%% [2]: [https://arrow.apache.org/docs/format/Glossary.html#term-physical-layout] -%% -%% [3]: [https://arrow.apache.org/docs/format/Glossary.html#term-type] -%% -%% [4]: [https://arrow.apache.org/docs/format/Columnar.html#null-count] -%% -%% [5]: [https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout] -%% -%% [6]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% -%% [7]: [https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout] -%% @end -module(arrow_array). + +%% TODO Check formatting with erlfmt +%% erlfmt:ignore +-moduledoc """ +Provides a record, a behaviour, and functions to work with Apache Arrow Arrays. + +This module as mentioned, provides conveniences for working with Apache Arrow +[Arrays](https://arrow.apache.org/docs/format/Glossary.html#term-array) of +different +[Layouts](https://arrow.apache.org/docs/format/Glossary.html#term-physical-layout). +Firstly, it provides a record to represent all the Layout. Secondly, it provides +a behaviour for different layouts to adhere to common functionality. Lastly, it +provides functions to work with Arrays in a common manner. + +# The Structure of an Array + +`arrow`'s implementation of an Array has the following fields in its record +definition, each corresponding to a field in the Arrow Columnar Format +specification: + +| Field | Type | Representing | +|-------------------|-----------------------------|-------------------------------------------------------------------------------------------| +| `layout` | `t:atom/0` | Layout | +| `type` | `t:arrow_type:arrow_type/0` | [Logical Type](https://arrow.apache.org/docs/format/Glossary.html#term-type) | +| `len` | `t:pos_integer/0` | [Length](https://arrow.apache.org/docs/format/Columnar.html#array-lengths) | +| `element_len` | `t:pos_integer/0` | Length of each element | +| `null_count` | `t:non_neg_integer/0` | [Null Count](https://arrow.apache.org/docs/format/Columnar.html#null-count) | +| `validity_bitmap` | `t:arrow_buffer:buffer/0` | [Validity Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) | +| `offsets` | `t:arrow_buffer:buffer/0` | [Offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) | +| `data` | `t:arrow_buffer:buffer/0` | Value Buffer | + + +Certain fields are not required for certain layouts. For example, for the +Fixed-Sized Primitive Layout, the offsets field is not required, in which case +it is assigned as `undefined`. Similarly, the `validity_bitmap` is not required +if there are no null values, in which case it is also assigned as `undefined`. + +# The Behaviour of an Array + +As of right now, a layout needs to implement the `c:from_erlang/2` callback. +This is then used in the `from_erlang/3` function to create new arrays. + +# Functions for working with Arrays + +As of right now, only functions to access the various fields, to create +new arrays, and to serialize arrays to arrow exist. +""". -export([ from_erlang/3, layout/1, @@ -113,29 +80,33 @@ -export_type([array/0, layout/0]). +-doc "Represents an Array.". -type array() :: #array{}. -%% Represents an Array. +-doc "Represents the Layout of an Array.". -type layout() :: fixed_primitive | variable_binary | fixed_list | variable_list. -%% Represents the Layout of an Array. %%%%%%%%%%%%%%%%%%%% %% Array Creation %% %%%%%%%%%%%%%%%%%%%% +-doc """ +Creates a new array of a certain layout, given its value and options from its +erlang representation. +""". -callback from_erlang(Value :: [arrow_type:native_type()], Opts :: map()) -> - Array :: #array{}. -%% Creates a new array of a certain layout, given its value and options from its -%% erlang representation. + Array :: array(). -%% @doc A common way to create a new array, given its layout, value, and options. -%% from its erlang representation. +-doc """ +A common way to create a new array, given its layout, value, and options. +from its erlang representation. +""". -spec from_erlang( Layout :: layout(), Value :: [arrow_type:native_type()], Opts :: map() | arrow_type:arrow_type() ) -> - Array :: #array{}. + Array :: array(). from_erlang(Layout, Value, Opts) -> case Layout of fixed_primitive -> arrow_fixed_primitive_array:from_erlang(Value, Opts); @@ -148,43 +119,43 @@ from_erlang(Layout, Value, Opts) -> %% Array Data and Metadata Access %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the layout of an array. --spec layout(Array :: #array{}) -> Layout :: layout(). +-doc "Returns the layout of an array.". +-spec layout(Array :: array()) -> Layout :: layout(). layout(Array) -> Array#array.layout. -%% @doc Returns the type of an array. --spec type(Array :: #array{}) -> Type :: arrow_type:arrow_type(). +-doc "Returns the type of an array.". +-spec type(Array :: array()) -> Type :: arrow_type:arrow_type(). type(Array) -> Array#array.type. -%% @doc Returns the length of an array. --spec len(Array :: #array{}) -> Length :: pos_integer(). +-doc "Returns the length of an array.". +-spec len(Array :: array()) -> Length :: pos_integer(). len(Array) -> Array#array.len. -%% @doc Returns the length of an array. --spec element_len(Array :: #array{}) -> Length :: pos_integer() | undefined. +-doc "Returns the length of an element in an array.". +-spec element_len(Array :: array()) -> Length :: pos_integer() | undefined. element_len(Array) -> Array#array.element_len. -%% @doc Returns the null count of an array. --spec null_count(Array :: #array{}) -> NullCount :: non_neg_integer(). +-doc "Returns the null count of an array.". +-spec null_count(Array :: array()) -> NullCount :: non_neg_integer(). null_count(Array) -> Array#array.null_count. -%% @doc Returns the validity bitmap of an array. --spec validity_bitmap(Array :: #array{}) -> ValidityBitmap :: #buffer{} | undefined. +-doc "Returns the validity bitmap of an array.". +-spec validity_bitmap(Array :: array()) -> ValidityBitmap :: arrow_buffer:buffer() | undefined. validity_bitmap(Array) -> Array#array.validity_bitmap. -%% @doc Returns the offsets of an array. --spec offsets(Array :: #array{}) -> Offsets :: #buffer{} | undefined. +-doc "Returns the offsets of an array.". +-spec offsets(Array :: array()) -> Offsets :: arrow_buffer:buffer() | undefined. offsets(Array) -> Array#array.offsets. -%% @doc Returns the data of an array. --spec data(Array :: #array{}) -> Data :: #buffer{} | #array{} | undefined. +-doc "Returns the data of an array.". +-spec data(Array :: array()) -> Data :: arrow_buffer:buffer() | array() | undefined. data(Array) -> Array#array.data. @@ -192,25 +163,27 @@ data(Array) -> %% Array Serialization %% %%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Serializes an array into the Arrow binary form. -%% -%% Serializes the buffers of an Array and concatenates them in the following -%% order: -%% -%%
    -%%
  1. `validity'
    2. -%%
  2. `offsets'
    3. -%%
  3. `data'
    4. -%%
-%% -%% In case an array doesn't have any of the following buffers, it is ommitted. -%% (e.g. validity in arrays with a null count of 0, offsets in fixed primitive -%% arrays). In the case of a nested array, `data' will be serialized form of -%% nested array. -%% -%% Do note that this is just binary form that includes the buffers in an Array, -%% and not IPC. --spec to_arrow(Array :: #array{}) -> Arrow :: binary(). +%% TODO Check formatting with erlfmt +%% erlfmt:ignore +-doc """ +Serializes an array into the Arrow binary form. + +Serializes the buffers of an Array and concatenates them in the following +order: + +1. `validity` +2. `offsets` +3. `data` + +In case an array doesn't have any of the following buffers, it is omitted. +(e.g. validity in arrays with a null count of 0, offsets in fixed primitive +arrays). In the case of a nested array, `data` will be serialized form of nested +array. + +Do note that this is just binary form that includes the buffers in an Array, +and not IPC. +""". +-spec to_arrow(Array :: array()) -> Arrow :: binary(). to_arrow(Array) -> Validity = some(validity_bitmap(Array)), Offsets = some(offsets(Array)), @@ -218,7 +191,7 @@ to_arrow(Array) -> <>. --spec some(Value :: #array{} | #buffer{} | undefined) -> Binary :: binary(). +-spec some(Value :: array() | arrow_buffer:buffer() | undefined) -> Binary :: binary(). some(undefined) -> <<>>; some(Buffer) when is_record(Buffer, buffer) -> diff --git a/src/arrow_array.hrl b/src/arrow_array.hrl index dbb5ae9..4446bf1 100644 --- a/src/arrow_array.hrl +++ b/src/arrow_array.hrl @@ -23,7 +23,7 @@ len :: pos_integer(), element_len :: pos_integer() | undefined, null_count :: non_neg_integer(), - validity_bitmap :: #buffer{} | undefined, - offsets :: #buffer{} | undefined, - data :: #buffer{} | #array{} + validity_bitmap :: arrow_buffer:buffer() | undefined, + offsets :: arrow_buffer:buffer() | undefined, + data :: arrow_buffer:buffer() | #array{} }). diff --git a/src/arrow_bitmap.erl b/src/arrow_bitmap.erl index 1747514..28b743c 100644 --- a/src/arrow_bitmap.erl +++ b/src/arrow_bitmap.erl @@ -15,57 +15,52 @@ % specific language governing permissions and limitations % under the License. -%% @doc Validity Bitmap implementation for `arrow'. -%% -%% Defines a function `validity_bitmap/1' to return the Validity Bitmap[1] along -%% with the Null Count[2], of an Array. -%% -%% An important thing to consider about our implementation of the Null Count is -%% that we need to support both `undefined' and `nil' as null values as they are -%% the conventions for null values in Erlang, and Elixir respectively. -%% -%% There are 5 important characteristics to remember about the validity bitmap: -%% -%%
    -%%
  1. -%% A null value is represented by a 0 bit, and a non null value by a 1 bit. -%%
    2. -%%
  2. -%% Every 8 elements's validities are batched into a byte, which are then -%% reversed as Arrow uses least-significant bit (LSB) numbering (more in -%% attached reference). -%%
    3. -%%
  3. -%% If a "batch" consists of less than 8 elements, its validity needs to be -%% padded by 0 bits so that it can make a byte. -%%
    4. -%%
  4. -%% Each byte is stored in a slot of a Buffer (see docs for -%% `arrow_buffer'). This buffer with the validities of each batch of 8 -%% elements make up what is called the Validity Bitmap. -%%
    5. -%%
  5. -%% If the Null Count is 0, we can allocate the Validity Bitmap as a NULL -%% pointer (which in Erlang's case is `undefined'). -%%
    6. -%%
-%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#null-count] -%% @end -module(arrow_bitmap). --export([validity_bitmap/1]). +-moduledoc """ +Validity Bitmap implementation for `arrow`. + +Defines a function `validity_bitmap/1` to return the [Validity +Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) +along with the [Null +Count](https://arrow.apache.org/docs/format/Columnar.html#null-count), of an +Array. + +An important thing to consider about our implementation of the Null Count is +that we need to support both `undefined` and `nil` as null values as they are +the conventions for null values in Erlang, and Elixir respectively. + +There are 5 important characteristics to remember about the validity bitmap: --include("arrow_buffer.hrl"). +1. A null value is represented by a 0 bit, and a non null value by a 1 bit. + +2. Every 8 elements's validities are batched into a byte, which are then + reversed as Arrow uses least-significant bit (LSB) numbering (more in + attached reference). + +3. If a "batch" consists of less than 8 elements, its validity needs to be + padded by 0 bits so that it can make a byte. + +4. Each byte is stored in a slot of a Buffer (see docs for `arrow_buffer`). This + buffer with the validities of each batch of 8 elements make up what is called + the Validity Bitmap. + +5. If the Null Count is 0, we can allocate the Validity Bitmap as a NULL pointer + (which in Erlang's case is `undefined`). +""". +-export([validity_bitmap/1]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Validity Bitmap & Null Count %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the Validity Bitmap along with the Null Count, of -%% an Array. +-doc """ +Returns the Validity Bitmap along with the Null Count, of an Array. +""". +%% TODO: Bitmap should be arrow_buffer:buffer() | undefined +%% Investigate why Dialyzer isn't raising, and potentially +%% run eqWAlizer via ELP on the CI. -spec validity_bitmap(Value :: [arrow_type:native_type()] | list()) -> - {Bitmap :: #buffer{}, non_neg_integer()}. + {Bitmap :: arrow_buffer:buffer(), non_neg_integer()}. validity_bitmap(Value) -> case (lists:member(undefined, Value)) orelse (lists:member(nil, Value)) of true -> @@ -78,7 +73,7 @@ validity_bitmap(Value) -> Value :: [arrow_type:native_type()], Acc :: binary(), NullCount :: non_neg_integer() -) -> {Bitmap :: #buffer{}, NullCount :: non_neg_integer()}. +) -> {Bitmap :: arrow_buffer:buffer(), NullCount :: non_neg_integer()}. bitmap([X1, X2, X3, X4, X5, X6, X7, X8 | Rest], Acc, NullCount) -> %% By assigning B8 as X1's validity, we are following LSB numbering. B8 = validity(X1), diff --git a/src/arrow_buffer.erl b/src/arrow_buffer.erl index 08ff561..ecec6d5 100644 --- a/src/arrow_buffer.erl +++ b/src/arrow_buffer.erl @@ -15,55 +15,52 @@ % specific language governing permissions and limitations % under the License. -%% @doc Buffer implementation for `arrow'. -%% This module adds suppport for buffers, or Contiguous Memory Regions. -%% -%% There are multiple things to know about buffers[1]: -%% -%%
    -%%
  1. -%% Each value it stores is called an element or a slot[2]. -%%
    2. -%%
  2. -%% Each slot's length (in bytes) is a positive integer. As a result when we -%% say that a slot has a length of 1, we mean that each slot has a length of -%% 1 byte. -%%
    3. -%%
  3. -%% The buffer's length in the metadata refers to the unpadded binary's size in bytes. -%%
    4. -%%
  4. -%% All buffers have a size that is a multiple of 64. If their data's length -%% is not a multiple of 64, it must be padded (in this implementation, by -%% zeros). -%%
    5. -%%
  5. -%% Null values are represented in this implementation by zeros. -%%
    6. -%%
  6. -%% In this implementation buffers can be initialized from raw bytes as data -%% apart datatypes supported by Arrow. This is so that the Validity Bitmap -%% Buffer can be initialized. -%%
    7. -%%
-%% -%% [1]: [https://arrow.apache.org/docs/format/Glossary.html#term-buffer] -%% -%% [2]: [https://arrow.apache.org/docs/format/Glossary.html#term-slot] -%% @end -module(arrow_buffer). + +%% TODO Check formatting with erlfmt +%% erlfmt:ignore +-moduledoc """ +Buffer implementation for `arrow`. + +This module adds support for buffers, or Contiguous Memory Regions. There are +multiple things to know about +[buffers](https://arrow.apache.org/docs/format/Glossary.html#term-buffer): + +1. Each value it stores is called an element or a + [slot](https://arrow.apache.org/docs/format/Glossary.html#term-slot). + +2. Each slot's length (in bytes) is a positive integer. As a result when we say + that a slot has a length of 1, we mean that each slot has a length of 1 byte. + +3. The buffer's length in the metadata refers to the unpadded binary's size in + bytes. + +4. All buffers have a size that is a multiple of 64. If their data's length is + not a multiple of 64, it must be padded (in this implementation, by zeros). + +5. Null values are represented in this implementation by zeros. + +6. In this implementation buffers can be initialized from raw bytes as data + apart datatypes supported by Arrow. This is so that the Validity Bitmap + Buffer can be initialized. +""". -export([from_erlang/2, from_erlang/3, to_arrow/1, to_erlang/1, size/1]). -include("arrow_buffer.hrl"). -%% @doc Creates a new buffer from a list of Erlang values or binaries, given its -%% type -%% @end +-export_type([buffer/0]). + +-doc "Represents an Arrow Buffer.". +-type buffer() :: #buffer{}. + +-doc """ +Creates a new buffer from a list of Erlang values or binaries, given its type. +""". -spec from_erlang( Value :: [arrow_type:native_type()] | binary(), Type :: arrow_type:arrow_longhand_type() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). from_erlang(Data, Type) -> Len = case Type of @@ -74,15 +71,16 @@ from_erlang(Data, Type) -> end, from_erlang(Data, Type, Len). -%% @doc Creates a new buffer from a list of Erlang values or binaries, given its -%% type and length -%% @end +-doc """ +Creates a new buffer from a list of Erlang values or binaries, given its +type and length +""". -spec from_erlang( Data :: [arrow_type:native_type()] | binary(), Type :: arrow_type:arrow_longhand_type(), DataLen :: pos_integer() | undefined ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). from_erlang(Data, Type, DataLen) -> Len = case Type of @@ -95,9 +93,8 @@ from_erlang(Data, Type, DataLen) -> end, #buffer{type = Type, length = Len, data = Data}. -%% @doc Returns an Arrow buffer binary given a buffer. -%% @end --spec to_arrow(Buffer :: #buffer{}) -> binary(). +-doc "Returns an Arrow buffer binary given a buffer.". +-spec to_arrow(Buffer :: arrow_buffer:buffer()) -> binary(). to_arrow(Buffer) when is_record(Buffer, buffer) -> Type = Buffer#buffer.type, Bin = @@ -112,16 +109,15 @@ to_arrow(Buffer) when is_record(Buffer, buffer) -> to_arrow(_Buffer) -> erlang:error(badarg). -%% @doc Returns a list of Erlang values or binaries from a buffer. -%% @end --spec to_erlang(Buffer :: #buffer{}) -> [arrow_type:native_type()]. +-doc "Returns a list of Erlang values or binaries from a buffer.". +-spec to_erlang(Buffer :: arrow_buffer:buffer()) -> [arrow_type:native_type()]. to_erlang(Buffer) when is_record(Buffer, buffer) -> Buffer#buffer.data; to_erlang(_Buffer) -> erlang:error(badarg). -%% @doc Returns the size of the buffer inclusive of padding in bytes. --spec size(Buffer :: #buffer{}) -> pos_integer(). +-doc "Returns the size of the buffer inclusive of padding in bytes.". +-spec size(Buffer :: arrow_buffer:buffer()) -> pos_integer(). size(Buffer) -> Len = Buffer#buffer.length * 8, round((Len + arrow_utils:pad_len(Len)) / 8). diff --git a/src/arrow_fixed_list_array.erl b/src/arrow_fixed_list_array.erl index 53a0d49..7308d7c 100644 --- a/src/arrow_fixed_list_array.erl +++ b/src/arrow_fixed_list_array.erl @@ -15,50 +15,46 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size List Layout. -%% -%% This module provides support for the Fixed-Size List Layout[1], which is an -%% layout that supports storing a list of lists of a specific length and -%% nesting. -%% -%% == Invalid Input == -%% -%% It is important that care is taken when passing input values to this module. -%% For performance reasons, the input is not validated. The function crashes on -%% nesting that is inconsitent: a. with the type, b. between elements. The lists -%% are checked to have the same length as each other at the top level, but not -%% for deeper levels. The behaviour of the module on invalid input -%% CANNOT BE GUARANTEED. Therefore, one must be careful to not -%% to CRASH THE PROCESS or worse still, PRODUCE INVALID -%% OUTPUT. -%% -%% Any input must follow the following rules: -%% -%%
    -%%
  1. The length of each element must be consistent with the type
    2. -%%
  2. The nesting of each element must be consistent with the type
    3. -%%
  3. The length of each element must be consistent with each other
    4. -%%
  4. The nesting of each element must be consistent with each other
    5. -%%
  5. -%% The nested type is a `fixed_list' (as only then can fixed size be -%% guaranteed) -%%
    6. -%%
-%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout -module(arrow_fixed_list_array). +-moduledoc """ +Provides support for Arrow's Fixed-Size List Layout. + +This module provides support for the [Fixed-Size List +Layout](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout), +which is a layout that supports storing a list of lists of a specific length and +nesting. + +## Invalid Input + +It is important that care is taken when passing input values to this module. For +performance reasons, the input is not validated. The function crashes on nesting +that is inconsistent: either with the type, or between elements. The lists are +checked to have the same length as each other at the top level, but not for +deeper levels. The behaviour of the module on invalid input **CANNOT BE +GUARANTEED**. Therefore, one must be careful to not to **CRASH THE PROCESS** or +worse still, **PRODUCE INVALID OUTPUT**. + +Any input must follow the following rules: + +1. The length of each element must be consistent with the type +2. The nesting of each element must be consistent with the type +3. The length of each element must be consistent with each other +4. The nesting of each element must be consistent with each other +5. The nested type is a `fixed_list` (as only then can fixed size be + guaranteed) +""". -behaviour(arrow_array). -export([from_erlang/2]). -include("arrow_array.hrl"). -%% @doc Creates a Fixed-Size List Array given the values and type. -%% -%% Accepts a map with the type, or the type directly. -%% @end +-doc """ +Creates a Fixed-Size List Array given the values and type. +Accepts a map with the type, or the type directly. +""". -spec from_erlang(Values :: list(), Type :: map() | arrow_type:arrow_type()) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Values, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined -> diff --git a/src/arrow_fixed_primitive_array.erl b/src/arrow_fixed_primitive_array.erl index 1f5bc5b..d37d31b 100644 --- a/src/arrow_fixed_primitive_array.erl +++ b/src/arrow_fixed_primitive_array.erl @@ -15,40 +15,30 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size Primitive Layout. -%% -%% The Primitive Layout[1] uses the following metadata in the `arrow_array': -%% -%%
    -%%
  1. -%% `layout', of type {@link atom()}, and a constant value of `fixed_primitive'. -%%
    2. -%%
  2. -%% `type', of type `t:arrow_type:arrow_primitive_type()', which -%% represents the Logical Type of the Array. -%%
    3. -%%
  3. `len', of type {@link pos_integer()}, which represents the Array's Length.
    4. -%%
  4. -%% `null_count', of type {@link non_neg_integer()}, which represents the Array's -%% Null Count, or the number of undefined values in the Array. -%%
    5. -%%
  5. -%% `validity_bitmap', which is a buffer (`arrow_buffer') or the atom -%% `undefined', which represents the Array's Validity Bitmap[2]. -%%
    6. -%%
  6. -%% `data', which is a buffer (`arrow_buffer'), which represents the Array's Value Buffer. -%%
    7. -%%
-%% -%% The `offsets' field will be allocated as `undefined'. Same goes for the -%% `validity_bitmap' field in the case of a 0 `null_count'. -%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% @end -module(arrow_fixed_primitive_array). +-moduledoc """ +Provides support for Arrow's Fixed-Size Primitive Layout. The [Primitive +Layout](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout) +uses the following metadata in the `arrow_array`: + +1. `layout`, of type `t:atom/0`, and a constant value of + `fixed_primitive`. +2. `type`, of type `t:arrow_type:arrow_primitive_type/0`, which + represents the Logical Type of the Array. +3. `len`, of type `t:pos_integer/0`, which represents the Array's + Length. +4. `null_count`, of type `t:non_neg_integer/0`, which represents + the Array's Null Count, or the number of undefined values in the + Array. +5. `validity_bitmap`, which is a buffer (`arrow_buffer`) or the + atom `undefined`, which represents the Array's [Validity + Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps). +6. `data`, which is a buffer (`arrow_buffer`), which represents the + Array's Value Buffer. + +The `offsets` field will be allocated as `undefined`. Same goes for the +`validity_bitmap` field in the case of a 0 `null_count`. +""". -behaviour(arrow_array). -export([from_erlang/2]). @@ -59,16 +49,17 @@ %% Array Creation %% %%%%%%%%%%%%%%%%%%%% -%% @doc Creates a new primitive array, given its value and type from its erlang -%% representation. -%% -%% Accepts a map with the type, or the type directly. -%% @end +-doc """ +Creates a new primitive array, given its value and type from its erlang +representation. + +Accepts a map with the type, or the type directly. +""". -spec from_erlang( Value :: [arrow_type:native_type()], Type :: map() | arrow_type:arrow_primitive_type() ) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Value, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined -> diff --git a/src/arrow_ipc_field.erl b/src/arrow_ipc_field.erl index 4ed6145..1b6938a 100644 --- a/src/arrow_ipc_field.erl +++ b/src/arrow_ipc_field.erl @@ -15,64 +15,64 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with individual fields in -%% a Schema -%% -%% A Field[1] represents a single column in a table (which is represented by a -%% Schema[2]). This module provides a record and a function to manage all the -%% metadata required to represent a column. Metadata such as: -%% -%%
    -%%
  1. -%% `name': The Name of the column. This can either be a string or null. -%%
    2. -%%
  2. -%% `nullable': A boolean representing whether a column can have null values. -%% Generally true. -%%
    3. -%%
  3. -%% `type': The Type of the column, and is of type -%% `arrow_ipc_type:ipc_type()' -%%
    4. -%%
  4. -%% `dictionary': A boolean representing if the column is dictionary encoded -%%
    5. -%%
  5. -%% `children': The child fields of a nested datatype -%%
    6. -%%
  6. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    7. -%%
-%% -%% Currently, dictionary encoding and custom metadata are not supported, but -%% they have been added for forwards comapatibility. -%% -%% [1]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492] -%% -%% [2]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530] -%% @end -module(arrow_ipc_field). +-moduledoc """ +Provides a record and functions to deal with individual fields in +a Schema + +A +[Field](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492) +represents a single column in a table (which is represented by a +[Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530). +This module provides a record and a function to manage all the metadata required +to represent a column. Metadata such as: + +1. `name`: The Name of the column. This can either be a string or + null. +2. `nullable`: A boolean representing whether a column can have null + values. Generally true. +3. `type`: The Type of the column, and is of type + `t:arrow_ipc_type:ipc_type/0` +4. `dictionary`: A boolean representing if the column is dictionary + encoded +5. `children`: The child fields of a nested datatype +6. `custom_metadata`: A list of custom metadata in key-value format + +Currently, dictionary encoding and custom metadata are not supported, but +they have been added for forwards comapatibility. +""". -export([from_erlang/1, from_erlang/2, from_erlang/3]). +-export_type([field/0]). -include("arrow_ipc_schema.hrl"). -%% @doc Creates a field given the type of the column. Assigns the name as -%% `undefined'. --spec from_erlang(Type :: arrow_ipc_type:ipc_type()) -> Field :: #field{}. +-doc """ +Represents an IPC Field. +""". +-type field() :: #field{}. + +-doc """ +Creates a field given the type of the column. Assigns the name as +`undefined`. +""". +-spec from_erlang(Type :: arrow_ipc_type:ipc_type()) -> Field :: field(). from_erlang(Type) -> from_erlang(Type, undefined, []). -%% @doc Creates a field given the type and name of the column. +-doc """ +Creates a field given the type and name of the column. +""". -spec from_erlang(Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined) -> - Field :: #field{}. + Field :: field(). from_erlang(Type, Name) -> from_erlang(Type, Name, []). -%% @doc Creates a field given the type, name and children of the column. +-doc """ +Creates a field given the type, name and children of the column. +""". -spec from_erlang( - Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined, Children :: [#field{}] + Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined, Children :: [field()] ) -> - Field :: #field{}. + Field :: field(). from_erlang(Type, Name, Children) -> #field{name = Name, type = Type, children = Children}. diff --git a/src/arrow_ipc_file.erl b/src/arrow_ipc_file.erl index ee45e28..f3eb82c 100644 --- a/src/arrow_ipc_file.erl +++ b/src/arrow_ipc_file.erl @@ -15,27 +15,32 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides records and functions to deal with the IPC File. -%% -%% The IPC File[1] is an extension of the IPC Stream[2] that supports random -%% access with the help of a footer which contains the offsets of all the -%% messages. -%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format -%% [2]: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format -%% @end -module(arrow_ipc_file). +-moduledoc """ +Provides records and functions to deal with the IPC File. The [IPC +File](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) is an +extension of the [IPC Stream]( +https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format) that +supports random access with the help of a footer which contains the offsets of +all the messages. +""". -export([from_erlang/2, to_ipc/1]). -include("arrow_ipc_message.hrl"). -include("arrow_ipc_file.hrl"). +-type file() :: #file{}. + %%%%%%%%%%%%%%%%%%% %% from_erlang/2 %% %%%%%%%%%%%%%%%%%%% -%% @doc Creates a file given a schema message and a list of record batch messages. --spec from_erlang(Schema :: #message{}, RecordBatches :: [#message{}]) -> #file{}. +-doc """ +Creates a file given a schema message and a list of record batch messages. +""". +-spec from_erlang( + Schema :: arrow_ipc_message:message(), RecordBatches :: [arrow_ipc_message:message()] +) -> file(). from_erlang(SchemaMsg, RecordBatches) -> SchemaEMF = arrow_ipc_message:to_ipc(SchemaMsg), {RecordBatchBlocks, EMFs} = blocks(byte_size(SchemaEMF), RecordBatches, [], []), @@ -49,7 +54,7 @@ from_erlang(SchemaMsg, RecordBatches) -> %% Returns the blocks as wells as the intermediate EMFs used to generate each block. -spec blocks( Offset :: non_neg_integer(), - Messages :: [#message{}], + Messages :: [arrow_ipc_message:message()], Blocks :: [#block{}], EMFs :: [binary()] ) -> {Blocks :: [#block{}], EMFs :: [binary()]}. @@ -68,8 +73,10 @@ blocks(Offset, [H | T], Blocks, EMFs) -> %% to_ipc/1 %% %%%%%%%%%%%%%% -%% @doc Serializes a file into the IPC File Format --spec to_ipc(File :: #file{}) -> SerializedFile :: binary(). +-doc """ +Serializes a file into the IPC File Format. +""". +-spec to_ipc(File :: file()) -> SerializedFile :: binary(). to_ipc(File) -> Footer = arrow_format_nif:serialize_footer(File#file.footer), Sz = byte_size(Footer), diff --git a/src/arrow_ipc_message.erl b/src/arrow_ipc_message.erl index 58c3f14..c27eb14 100644 --- a/src/arrow_ipc_message.erl +++ b/src/arrow_ipc_message.erl @@ -15,73 +15,79 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with the Encapsulated Message -%% Format. -%% -%% A Message is a serialized form of a Schema[1] or a RecordBatch[2] (which may -%% be required to read a serialized array) along with some metadata. This module -%% provides a record and a function to manage all the metadata required to -%% represent a message. Metadata such as: -%% -%%
    -%%
  1. -%% `version': The Apache Arrow Format Version. One of v1..v5. Defaults to v5. -%%
    2. -%%
  2. -%% `header': The metadata of the Schema or RecordBatch -%%
    3. -%%
  3. -%% `body_length': The length of the body in bytes -%%
    4. -%%
  4. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    5. -%%
  5. -%% `body': The actual body. Can be undefined (in the case of Schema) or a -%% binary (in the case of Record Batch). -%%
    6. -%%
-%% -%% Currently, changing the version and custom metadata are not supported, but -%% they have been added for forwards comapatibility. -%% -%% This module also provides the `to_ipc/1' function which serializes the -%% message into the Encapsulated Message Format[3]. However, this function gives -%% incomplete output with invalid metadata, because of an unsatisfied -%% dependency on flatbuffers, which is required for serializing the metadata. -%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#schema-message] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message] -%% -%% [3]: [https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format] -%% @end -module(arrow_ipc_message). +-moduledoc """ +Provides a record and functions to deal with the Encapsulated Message +Format. + +A Message is a serialized form of a +[Schema](https://arrow.apache.org/docs/format/Columnar.html#schema-message) or a +[RecordBatch](https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message) +(which may be required to read a serialized array) along with some metadata. +This module provides a record and a function to manage all the metadata required +to represent a message. Metadata such as: + +1. `version`: The Apache Arrow Format Version. One of `v1..v5`. + Defaults to v5. +2. `header`: The metadata of the Schema or RecordBatch +3. `body_length`: The length of the body in bytes +4. `custom_metadata`: A list of custom metadata in key-value format +5. `body`: The actual body. Can be undefined (in the case of Schema) + or a binary (in the case of Record Batch). + +Currently, changing the version and custom metadata are not supported, but they +have been added for forwards compatibility. + +This module also provides the `to_ipc/1` function which serializes the message +into the [Encapsulated Message +Format](https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format). +However, this function gives *incomplete output* with invalid metadata, because +of an unsatisfied dependency on flatbuffers, which is required for serializing +the metadata. +""". -export([from_erlang/1, from_erlang/2, to_ipc/1, to_stream/1, metadata_len/1, body_from_erlang/1]). --export_type([metadata_version/0, key_value/0]). +-export_type([message/0, metadata_version/0, key_value/0]). -include("arrow_ipc_message.hrl"). +-doc "Represents a Message.". +-type message() :: #message{}. + +-doc """ +The Arrow version. See the +[definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L28-L49) +for more info: +""". -type metadata_version() :: v1 | v2 | v3 | v4 | v5. -%% The Arrow version. See the definition for more info: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L28-L49] +-doc """ +Key-Value structure for custom metadata. See the +[definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L432-L439) +for more info: +""". -type key_value() :: #{key => string(), value => string()}. -%% Key-Value structure for custom metadata. See the definition for more info: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L432-L439] -%% @doc Creates a message given a data header. --spec from_erlang(Header :: #schema{} | #record_batch{}) -> Message :: #message{}. +-doc """ +Creates a message given a data header. +""". +-spec from_erlang(Header :: arrow_ipc_schema:schema() | arrow_ipc_record_batch:record_batch()) -> + Message :: message(). from_erlang(Header) -> #message{header = Header, body_length = 0}. -%% @doc Creates a message given a data header and a body. --spec from_erlang(Header :: #schema{} | #record_batch{}, Body :: binary()) -> Message :: #message{}. +-doc """ +Creates a message given a data header and a body. +""". +-spec from_erlang( + Header :: arrow_ipc_schema:schema() | arrow_ipc_record_batch:record_batch(), Body :: binary() +) -> Message :: message(). from_erlang(Header, Body) -> #message{header = Header, body = Body, body_length = byte_size(Body)}. -%% @doc Serializes a message into the Encapsulated Message Format. --spec to_ipc(Message :: #message{}) -> EMF :: binary(). +-doc """ +Serializes a message into the Encapsulated Message Format. +""". +-spec to_ipc(Message :: message()) -> EMF :: binary(). to_ipc(Message) -> %% 0xFFFFFFFF in int32 Continuation = <<-1:32>>, @@ -98,8 +104,10 @@ to_ipc(Message) -> <>. -%% @doc Serializes a list of messages or EMFs into a Stream. --spec to_stream(Messages :: [#message{}] | [binary()]) -> Stream :: binary(). +-doc """ +Serializes a list of messages or EMFs into a Stream. +""". +-spec to_stream(Messages :: [message()] | [binary()]) -> Stream :: binary(). to_stream([H | _] = Messages) -> Msgs = if @@ -117,7 +125,9 @@ to_stream([H | _] = Messages) -> %% metadata_len/1 %% %%%%%%%%%%%%%%%%%%%% -%% @doc Returns the Metadata Length of an EMF. +-doc """ +Returns the Metadata Length of an EMF. +""". -spec metadata_len(EMF :: binary()) -> MetadataLen :: non_neg_integer(). metadata_len(EMF) -> <<_:32, MetadataLen:32, _Rest/binary>> = EMF, @@ -127,13 +137,14 @@ metadata_len(EMF) -> %% body_from_erlang/1 %% %%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the body of message from a list of arrays. -%% -%% Shorthand for: -%% ``` -%% <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>> -%% ''' -%% @end --spec body_from_erlang(Columns :: [#array{}]) -> Body :: binary(). +-doc """ +Returns the body of message from a list of arrays. + +Shorthand for: +``` +<<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>> +``` +""". +-spec body_from_erlang(Columns :: [arrow_array:array()]) -> Body :: binary(). body_from_erlang(Columns) -> <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>>. diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index 8e7fac8..6d1cd77 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -15,57 +15,49 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with RecordBatches -%% -%% A RecordBatch[1] represents a list of equal length arrays and their -%% coresponding buffers. This module provides a record and a function to manage -%% all the metadata required to represent a RecordBatch. Metadata such as: -%% -%%
    -%%
  1. -%% `length': The number of rows or records. In other words, the length of -%% an array. -%%
    2. -%%
  2. -%% `nodes': A list of maps, where each map has the length and null count -%% of an array -%%
    3. -%%
  3. -%% `buffers': A list of maps, where each map has the length and the offset -%% (from the beginning of the message body) of a buffer of an array. -%%
    4. -%%
  4. -%% `compression': The compression applied on the body of the Record Batch. -%% Can either by `undefined' (i.e. no compression), `zstd' for Zstandard[2], -%% or `lz4_frame' for LZ4 Frame[3]. Defaults to `undefined'. -%%
    5. -%%
-%% -%% Currently, compression is not supported, but it has been added for forwards -%% comapatibility. -%% -%% You can find RecordBatches in the Arrow spec here[4]. -%% -%% [1]: [https://github.com/apache/arrow/blob/16328f0ccc73b7df665b4a18feb6adf26b7aa0e2/format/Message.fbs#L81-L102] -%% -%% [2]: [https://facebook.github.io/zstd/] -%% -%% [3]: [https://android.googlesource.com/platform/external/lz4/+/HEAD/doc/lz4_Frame_format.md] -%% -%% [4]: [https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message] -%% @end -module(arrow_ipc_record_batch). +-moduledoc """ +Provides a record and functions to deal with RecordBatches. + + +A [RecordBatch](https://github.com/apache/arrow/blob/16328f0ccc73b7df665b4a18feb6adf26b7aa0e2/format/Message.fbs#L81-L102) represents a list of equal length arrays and their +corresponding buffers. This module provides a record and a function to manage +all the metadata required to represent a RecordBatch. Metadata such as: + +1. `length`: The number of rows or records. In other words, the + length of an array. +2. `nodes`: A list of maps, where each map has the length and null + count of an array +3. `buffers`: A list of maps, where each map has the length and the + offset (from the beginning of the message body) of a buffer of an + array. + +4. `compression`: The compression applied on the body of the Record + Batch. Can either be `undefined` (i.e. no compression), `zstd` for + [Zstandard](https://facebook.github.io/zstd/), or `lz4_frame` for [LZ4 + Frame](https://android.googlesource.com/platform/external/lz4/+/HEAD/doc/lz4_Frame_format.md). + Defaults to `undefined`. + +Currently, compression is not supported, but it has been added for forwards +comapatibility. + +You can find RecordBatches in the Arrow spec +[here](https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message). +""". -export([from_erlang/1]). --export_type([field_node/0, buffer/0]). +-export_type([field_node/0, buffer/0, record_batch/0]). -include("arrow_ipc_record_batch.hrl"). -include("arrow_array.hrl"). -type field_node() :: #{length => pos_integer(), null_count => non_neg_integer()}. -type buffer() :: #{offset => non_neg_integer(), length => pos_integer()}. +-type record_batch() :: #record_batch{}. -%% @doc Creates a RecordBatch given a list of arrays --spec from_erlang(Arrays :: [#array{}]) -> RecordBatch :: #record_batch{}. +-doc """ +Creates a RecordBatch given a list of arrays +""". +-spec from_erlang(Arrays :: [arrow_array:array()]) -> RecordBatch :: record_batch(). from_erlang(Arrays) -> FieldNodes = lists:map( fun(X) -> #{length => X#array.len, null_count => X#array.null_count} end, Arrays @@ -83,7 +75,7 @@ from_erlang(Arrays) -> #record_batch{length = Length, nodes = FieldNodes, buffers = lists:reverse(Buffers)}. %% Returns the buffer data for all the buffers and nested arrays in an array. --spec array_data(Array :: #array{}, CurOffset :: non_neg_integer()) -> +-spec array_data(Array :: arrow_array:array(), CurOffset :: non_neg_integer()) -> {[buffer()], NewOffset :: non_neg_integer()}. array_data(Array, CurOffset) -> Bitmap = Array#array.validity_bitmap, @@ -115,7 +107,7 @@ array_data(Array, CurOffset) -> {Buffers ++ [DataBuffer], NewOffset} end. --spec buffer_data(Buffer :: #buffer{}, CurOffset :: non_neg_integer()) -> +-spec buffer_data(Buffer :: arrow_buffer:buffer(), CurOffset :: non_neg_integer()) -> {BufferData :: buffer(), NewOffset :: pos_integer()}. buffer_data(Buffer, CurOffset) -> { diff --git a/src/arrow_ipc_schema.erl b/src/arrow_ipc_schema.erl index 22632fd..072a02a 100644 --- a/src/arrow_ipc_schema.erl +++ b/src/arrow_ipc_schema.erl @@ -15,58 +15,49 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with Schemas -%% -%% A Schema[1] represents a table, or a list of arrays of equal length. This -%% module provides a record and a function to manage all the metadata required -%% to represent a schema. Metadata such as: -%% -%%
    -%%
  1. -%% `endianness': The Endianness of the table. One of `little' or `big'. -%% Defaults to `little'. -%%
    2. -%%
  2. -%% `fields': The list of fields[2] in a table. -%%
    3. -%%
  3. -%% `type': The Layout of the column -%%
    4. -%%
  4. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    5. -%%
  5. -%% `features': Any features used by the table which may not be present in -%% other implementations of Arrow. -%%
    6. -%%
-%% -%% Currently, big endianness, custom metadata and features are not supported, -%% but they have been added for forwards comapatibility. -%% -%% You can find Schemas in the Arrow spec here[3]. -%% -%% [1]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530] -%% -%% [2]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492] -%% -%% [3]: [https://arrow.apache.org/docs/format/Columnar.html#schema-message] -%% @end -module(arrow_ipc_schema). +-moduledoc """ +Provides a record and functions to deal with Schemas. + +A [Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530) +represents a table, or a list of arrays of equal length. This module provides a +record and a function to manage all the metadata required to represent a schema. +Metadata such as: + +1. `endianness`: The Endianness of the table. One of `little` or + `big`. Defaults to `little`. +2. `fields`: The list of [fields](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492) in a table. +3. `type`: The Layout of the column +4. `custom_metadata`: A list of custom metadata in key-value format +5. `features`: Any features used by the table which may not be + present in other implementations of Arrow. + +Currently, big endianness, custom metadata and features are not supported, but +they have been added for forwards compatibility. + +You can find Schemas in the Arrow spec +[here](https://arrow.apache.org/docs/format/Columnar.html#schema-message). +""". -export([from_erlang/1]). --export_type([endianness/0, feature/0]). +-export_type([endianness/0, feature/0, schema/0]). -include("arrow_ipc_schema.hrl"). +-doc "Endianness of the data. Either `little` or `big`.". -type endianness() :: little | big. -%% Endianness of the data. Either `little' or `big'. +-doc """ +Features used in the data which may not be present in other implementations. +See the [definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L51-L78). +""". -type feature() :: unused | dictionary_replacement | compressed_body. -%% Features used in the data which may not be present in other implementations. -%% See the definition: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L51-L78] -%% @doc Creates a Schema given an ordered list of fields. --spec from_erlang(Fields :: [#field{}]) -> Schema :: #schema{}. +-doc "Represents a schema". +-type schema() :: #schema{}. + +-doc """ +Creates a Schema given an ordered list of fields. +""". +-spec from_erlang(Fields :: [arrow_ipc_field:field()]) -> Schema :: schema(). from_erlang(Fields) -> #schema{fields = Fields}. diff --git a/src/arrow_ipc_type.erl b/src/arrow_ipc_type.erl index f4b1e7c..def4957 100644 --- a/src/arrow_ipc_type.erl +++ b/src/arrow_ipc_type.erl @@ -15,20 +15,18 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides types and functions to work with IPC types. -%% -%% This module provides functions and types to produce the types in IPC Schema -%% definitions[1]. These types are generated according to these[2] definitions. -%% The types have been represented in the form `{TypeName, Metadata}', where -%% `TypeName' is the name of the type and is an atom, and `Metadata' is a map of -%% all the metadata associated with it. In case a type has no metadata -%% associated with it, it is represented as just `TypeName' -%% -%% [1]:https://github.com/apache/arrow/blob/main/format/Schema.fbs -%% -%% [2]: https://github.com/apache/arrow/blob/main/format/Schema.fbs#L82-L430 -%% @end -module(arrow_ipc_type). +-moduledoc """ +Provides types and functions to work with IPC types. + +This module provides functions and types to produce the types in IPC Schema +[definitions](https://github.com/apache/arrow/blob/main/format/Schema.fbs). +These types are generated according to `Type` union in the schema. The types +have been represented in the form `{TypeName, Metadata}`, where `TypeName` is +the name of the type and is an atom, and `Metadata` is a map of all the metadata +associated with it. In case a type has no metadata associated with it, it is +represented as just `TypeName`. +""". -export([from_erlang/1]). -export_type([ ipc_type/0, @@ -90,7 +88,9 @@ %% from_erlang/1 %% %%%%%%%%%%%%%%%%%%% -%% @doc Returns the IPC Type for an `#array{}'. +-doc """ +Returns the IPC Type for an `t:arrow_array:array/0`. +""". -spec from_erlang(Array :: arrow_array:array()) -> Type :: ipc_type(). from_erlang(Array) -> case Array#array.layout of diff --git a/src/arrow_offsets.erl b/src/arrow_offsets.erl index 73461c7..9fef0b0 100644 --- a/src/arrow_offsets.erl +++ b/src/arrow_offsets.erl @@ -15,68 +15,66 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Apache Arrow's Offsets -%% -%% Arrow has a concept of offsets[1] in order to tell the length of a slot[2] or -%% a single element in an array of variable-size elements. This module provides -%% support for generating offsets. -%% -%% There are couple of things to remember about offsets: -%% -%%
    -%%
  1. -%% Each element in the offsets coresponds to the distance in bytes of the -%% coresponding element in the values from the beginning of the buffer. -%% -%% I.E., distance of values[j] from beginning of the buffer = offsets[j]. -%%
    2. -%%
  2. -%% The very last element in the offsets is the length of the buffer, or -%% distance of the end of the last slot from the beginning of the buffer. -%%
    3. -%%
  3. -%% Thus, the offsets is one element longer than the values, or: -%% -%% length(offsets) == length(values) + 1 -%%
    4. -%%
  4. -%% Therefore, in order to find the length of a slot, we subtract the offset -%% the current element from the offset of the next element, or: -%% -%% slot[j] = offsets[j + 1] - offsets[j] -%%
    5. -%%
  5. -%% Null values have an offset of 0 as they take no memory in the buffer. -%% Thus, the previous offset and the current offset are equivalent if the -%% current element is a null. -%%
    6. -%%
-%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#terminology] -%% @end -module(arrow_offsets). --export([new/2, new/3]). +-moduledoc """ +Provides support for Apache Arrow's Offsets + +Arrow has a concept of +[offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) +in order to tell the length of a +[slot](https://arrow.apache.org/docs/format/Columnar.html#terminology) or a +single element in an array of variable-size elements. This module provides +support for generating offsets. + +There are couple of things to remember about offsets: + +Firstly, each element in the offsets corresponds to the distance in bytes of the +corresponding element in the values from the beginning of the buffer. + +I.E., distance of `values[j]` from beginning of the buffer = `offsets[j]`. + +Secondly, the very last element in the offsets is the length of the buffer, or +distance of the end of the last slot from the beginning of the buffer. --include("arrow_buffer.hrl"). +Thus, the offsets is one element longer than the values, or: + +``` +length(offsets) == length(values) + 1 +``` + +Therefore, in order to find the length of a slot, we subtract the offset the +current element from the offset of the next element, or: + +``` +slot[j] = offsets[j + 1] - offsets[j] +``` + +Finally, null values have an offset of 0 as they take no memory in the buffer. +Thus, the previous offset and the current offset are equivalent if the current +element is a null. +""". +-export([new/2, new/3]). -%% @doc Returns the offsets array given some values and their type as a buffer. +-doc """ +Returns the offsets array given some values and their type as a buffer. +""". -spec new( Value :: [arrow_type:native_type()], Type :: arrow_type:arrow_longhand_type() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). new(Values, Type) -> new(Values, Type, length(Values)). -%% @doc Returns the offsets array given some values, their type and length as a buffer. +-doc """ +Returns the offsets array given some values, their type and length as a buffer. +""". -spec new( Value :: [arrow_type:native_type()], Type :: arrow_type:arrow_longhand_type(), Length :: pos_integer() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). new(Values, Type, Len) -> Offsets = offsets(Values, [0], 0, Type), arrow_buffer:from_erlang(Offsets, {s, 32}, Len + 1). diff --git a/src/arrow_type.erl b/src/arrow_type.erl index f7a92b0..899e96c 100644 --- a/src/arrow_type.erl +++ b/src/arrow_type.erl @@ -15,123 +15,94 @@ % specific language governing permissions and limitations % under the License. -%%%----------------------------------------------------------------------------- -%% @doc Provides representation for as well as functions for working with Arrow -%% specific datatypes. -%% -%% This module provides a longhand (a 2 tuple) as well as a shorthand (an atom) -%% represenations for representing a primitive types. Additionally it represents -%% Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types -%% that are supported by this package). -%% -%% Generally, when we say an array A has type X, we mean that each element in A -%% has type X. Thus, an array of type of `Int8' would look like `[1, 2, 3]', but -%% an array of type `List' would look like `[[1, 2], [3]]'. -%% -%% == Primitive Types == -%% -%% A Primitive Type is any type that refers to a single value (as opposed to a -%% list of values). Currently it includes Booleans, Signed Integers, Unsigned -%% Integers, Floats and Binaries. Both a longhand and shorthand syntax had been -%% provided to initiate a type. -%% -%% The following is a comprehensive list of all supported primitive types: -%% -%%
    -%%
  1. Booleans
    2. -%%
  2. Signed Integers: Includes `Int 8', `Int 16', `Int 32' and `Int 64'
    3. -%%
  3. -%% Unsigned Integers: Includes `UInt 8', `UInt 16', `UInt 32' and `UInt 64' -%%
    4. -%%
  4. Floats: Includes `Float 16', `Float 32' and `Float 64'
    5. -%%
  5. Binaries
    6. -%%
-%% -%% === Shorthand Syntax === -%% -%% In the shorthand syntax, you can just use a single atom to refer to a type, -%% such that the atom is `'. The name is a single letter that refers -%% to the type, and the size is that type's size in bits. Take the example of -%% `Int 8'. Its name is `s', and size is `8'. Thus in shorthand, it is `s8'. In -%% the case of datatypes without variations, such as Booleans and Binaries, they -%% are represented by just their datatype. Do note that internally the longhand -%% syntax is used. -%% -%% Here you can find the name of each type: -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%%
TypeNameExample Atom
BooleansN/A`bool'
Signed Integer`s'`s64'
Unsigned Integer`u'`u16'
Float`f'`f32'
BinariesN/A`bin'
-%% -%% Types that are shorthand are under the `t:arrow_shorthand_type()' and -%% `arrow_short_*()' types. -%% -%% === Longhand Syntax === -%% -%% In longhand syntax, you use a 2 tuple to to refer to the type, such that the -%% tuple is `{Name, Size}'. Thus, `Int 8' would be represented as `{s, 8}'. Do -%% note that in the case or Booleans and Binaries their `Size' is `undefined', as -%% the have variable or undefined size. -%% -%% Types that are longhand are under the `t:arrow_longhand_type()' and -%% `arrow_long_*()' types. -%% -%% == Nested Type == -%% -%% A Nested Type is any data structure that supports nesting. This can include -%% Lists, FixedLists, Maps, Structs and more, but is curently limited to -%% FixedLists. Nested Types are represented using a 3 tuple: `{NestedType, Type, -%% Length}'. `NestedType' refers to the data structure that is nesting your -%% primitive type. `Type' refers to the type that is being nested in the data -%% structure. Do note that `Type' can be a Primitive Type, or another Nested -%% Type. Both the shorthand and the longhand syntax can be used or Primitive -%% Types.`Length' refers to the length of each element, and will only be -%% required to represent FixedLists. -%% -%% It is important to remember while nesting that the type refers to the type of -%% each element in an array. A list of `Int8' is 2 dimensional, and a list of -%% `List' is 3 dimensional. -%% -%% == Native Type == -%% -%% A Native Type is any Erlang Datatype that can be serialized by `arrow', -%% and is represented by the type `t:native_type()'. It currently only includes -%% values that are primitive in nature such as booleans, integers, floats, -%% binaries, and additionally the atoms `undefined' and `nil' to represent a -%% `NULL' value. -%% -%% @end -%%%----------------------------------------------------------------------------- -module(arrow_type). +-moduledoc """ +Provides representation for as well as functions for working with Arrow +specific datatypes. + +This module provides longhand (a 2 tuple) as well as shorthand (an atom) +representations for representing primitive types. Additionally it represents +Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types that +are supported by this package). + +Generally, when we say an array A has type X, we mean that each element in A +has type X. Thus, an array of type of `Int8` would look like `[1, 2, 3]`, but +an array of type `List` would look like `[[1, 2], [3]]`. + +## Primitive Types + +A Primitive Type is any type that refers to a single value (as opposed to a +list of values). Currently it includes Booleans, Signed Integers, Unsigned +Integers, Floats and Binaries. Both a longhand and shorthand syntax had been +provided to initiate a type. + +The following is a comprehensive list of all supported primitive types: + +1. Booleans +2. Signed Integers: Includes `Int 8`, `Int 16`, `Int 32` and + `Int 64` +3. Unsigned Integers: Includes `UInt 8`, `UInt 16`, `UInt 32` and + `UInt 64` +4. Floats: Includes `Float 16`, `Float 32` and `Float 64` +5. Binaries + +### Shorthand Syntax + +In the shorthand syntax, you can just use a single atom to refer to a type, +such that the atom is ``. The name is a single letter that refers +to the type, and the size is that type's size in bits. Take the example of +`Int 8`. Its name is `s`, and size is `8`. Thus in shorthand, it is `s8`. In +the case of datatypes without variations, such as Booleans and Binaries, they +are represented by just their datatype. Do note that internally the longhand +syntax is used. + +Here you can find the name of each table: + +| Type | Name | Example Atom | +|------------------|-------|--------------| +| Booleans | N/A | `bool` | +| Signed Integer | `s` | `s64` | +| Unsigned Integer | `u` | `u16` | +| Float | `f` | `f32` | +| Binaries | N/A | `bin` | + +Types that are shorthand are under the `t:arrow_shorthand_type/0` and +`arrow_short_*()` types. + +### Longhand Syntax + +In longhand syntax, you use a 2 tuple to to refer to the type, such that the +tuple is `{Name, Size}`. Thus, `Int 8` would be represented as `{s, 8}`. Do +note that in the case or Booleans and Binaries their `Size` is `undefined`, as +the have variable or undefined size. + +Types that are longhand are under the `t:arrow_longhand_type/0` and +`arrow_long_*()` types. + +### Nested Types + +A Nested Type is any data structure that supports nesting. This can include +Lists, FixedLists, Maps, Structs and more, but is curently limited to +FixedLists. Nested Types are represented using a 3 tuple: `{NestedType, Type, +Length}`. `NestedType` refers to the data structure that is nesting your +primitive type. `Type` refers to the type that is being nested in the data +structure. Do note that `Type` can be a Primitive Type, or another Nested +Type. Both the shorthand and the longhand syntax can be used or Primitive +Types.`Length` refers to the length of each element, and will only be +required to represent FixedLists. + +It is important to remember while nesting that the type refers to the type of +each element in an array. A list of `Int8` is 2 dimensional, and a list of +`List` is 3 dimensional. + +## Native Type + +A Native Type is any Erlang Datatype that can be serialized by `arrow`, +and is represented by the type `t:native_type/0`. It currently only includes +values that are primitive in nature such as booleans, integers, floats, +binaries, and additionally the atoms `undefined` and `nil` to represent a +`NULL` value. +""". -export([normalize/1, bit_length/1, byte_length/1, serialize/2]). -export_type([ arrow_type/0, @@ -354,7 +325,9 @@ normalize(_Type) -> erlang:error(badarg). -spec bit_length(Type :: arrow_type()) -> Length :: pos_integer() | undefined. -%% @doc Returns the size of the type in bits. +-doc """ +Returns the size of the type in bits. +""". bit_length(Type) when is_atom(Type) -> bit_length(normalize(Type)); bit_length({Type, Size}) when (Type =:= s) orelse (Type =:= u) orelse (Type =:= f) -> @@ -367,7 +340,9 @@ bit_length(Type) when tuple_size(Type) =:= 3 -> erlang:error(badarg). -spec byte_length(Type :: arrow_type()) -> Length :: pos_integer() | undefined. -%% @doc Returns the size of the type in bytes. +-doc """ +Returns the size of the type in bytes. +""". byte_length(Type) when is_atom(Type) -> byte_length(normalize(Type)); byte_length(Type) when tuple_size(Type) =:= 3 -> diff --git a/src/arrow_utils.erl b/src/arrow_utils.erl index ff45a22..a32caea 100644 --- a/src/arrow_utils.erl +++ b/src/arrow_utils.erl @@ -15,8 +15,8 @@ % specific language governing permissions and limitations % under the License. -%% @private Utils module -module(arrow_utils). +-moduledoc false. -export([nesting/1, flatten/1, flatten/3, pad_len/1, pad_len/2]). %% Finds the nesting level of deep list diff --git a/src/arrow_variable_binary_array.erl b/src/arrow_variable_binary_array.erl index 574b150..537d76a 100644 --- a/src/arrow_variable_binary_array.erl +++ b/src/arrow_variable_binary_array.erl @@ -15,26 +15,34 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Variable-Sized Binary Layout. -%% -%% The Variable-Sized Binary Layout[1] provides support for storing binaries of -%% varying length in a way similar to the primitive layout, i.e. in a 1 -%% Dimensional Array. -module(arrow_variable_binary_array). +-moduledoc """ +Provides support for Arrow's Variable-Sized Binary Layout. + +The [Variable-Sized Binary +Layout](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) +provides support for storing binaries of varying length in a way similar to the +primitive layout, i.e. in a 1 Dimensional Array. +""". -behaviour(arrow_array). -export([from_erlang/1, from_erlang/2]). -include("arrow_array.hrl"). -%% @doc Creates a Variable-Sized Binary Array given the values and options in the form of -%% a map, from its erlang representation. --spec from_erlang(Values :: [arrow_type:native_type()], Opts :: map()) -> Array :: #array{}. +-doc """ +Creates a Variable-Sized Binary Array given the values and options in the form of +a map, from its erlang representation. +""". +-spec from_erlang(Values :: [arrow_type:native_type()], Opts :: map()) -> + Array :: arrow_array:array(). from_erlang(Values, _Opts) -> from_erlang(Values). -%% @doc Creates a Variable-Sized Binary Array given the values --spec from_erlang(Values :: [arrow_type:native_type()]) -> Array :: #array{}. +-doc """ +Creates a Variable-Sized Binary Array given the values +""". +-spec from_erlang(Values :: [arrow_type:native_type()]) -> Array :: arrow_array:array(). from_erlang(Values) -> Len = length(Values), {Bitmap, NullCount} = arrow_bitmap:validity_bitmap(Values), diff --git a/src/arrow_variable_list_array.erl b/src/arrow_variable_list_array.erl index 79fc6af..630954d 100644 --- a/src/arrow_variable_list_array.erl +++ b/src/arrow_variable_list_array.erl @@ -15,30 +15,29 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size List Layout. -%% -%% This module provides support for the Fixed-Size List Layout[1], which is an -%% layout that supports storing a list of lists of a specific length and -%% nesting. -%% -%% == Invalid Input == -%% -%% It is important that care is taken when passing input values to this module. -%% For performance reasons, the input is not validated. The function crashes on -%% nesting that is inconsitent: a. with the type, b. between elements. The -%% behaviour on invalid input CANNOT BE GUARANTEED. Therefore, -%% one must be careful to not to CRASH THE PROCESS or worse -%% still, PRODUCE INVALID OUTPUT. -%% -%% Any input must follow the following rules: -%% -%%
    -%%
  1. The nesting of each element must be consistent with the type
    2. -%%
  2. The nesting of each element must be consistent with each other
    3. -%%
-%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout -module(arrow_variable_list_array). +-moduledoc """ +Provides support for Arrow's Variable-Size List Layout. + +This module provides support for the [Variable-Size List +Layout](https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout), +which is an layout that supports storing a list of lists of a specific length +and nesting. + +## Invalid Input + +It is important that care is taken when passing input values to this +module. For performance reasons, the input is not validated. The +function crashes on nesting that is inconsistent: a. with the type, b. +between elements. The behaviour on invalid input **CANNOT BE +GUARANTEED**. Therefore, one must be careful to not to **CRASH THE +PROCESS** or worse still, **PRODUCE INVALID OUTPUT**. + +Any input must follow the following rules: + +1. The nesting of each element must be consistent with the type +2. The nesting of each element must be consistent with each other +""". -behaviour(arrow_array). @@ -47,7 +46,7 @@ -include("arrow_array.hrl"). -spec from_erlang(Values :: list(), Type :: map() | arrow_type:arrow_type()) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Values, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined ->