From 638c6bfb02814ea56bd4bbf1f2e4c3baaef1c462 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 11:39:29 +0530 Subject: [PATCH 01/18] Update Erlang/OTP, Rebar3 and ExDoc --- .github/workflows/docs.yml | 4 ++-- .github/workflows/erlang.yml | 12 ++++++------ rebar.config | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6eba671..ce0d597 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,8 +51,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: 29.0 + rebar3-version: 3.27.0 - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml index 06ff608..b3db799 100644 --- a/.github/workflows/erlang.yml +++ b/.github/workflows/erlang.yml @@ -43,8 +43,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: 29.0 + rebar3-version: 3.27.0 - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -81,8 +81,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: 29.0 + rebar3-version: 3.27.0 - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -122,8 +122,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 25.1.0 - rebar3-version: '3.18.0' + otp-version: 29.0 + rebar3-version: 3.27.0 - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/rebar.config b/rebar.config index 5a757fa..46065cd 100644 --- a/rebar.config +++ b/rebar.config @@ -29,7 +29,7 @@ ]}. {plugins, [rebar3_cargo]}. -{project_plugins, [erlfmt, {rebar3_ex_doc, "0.2.18"}]}. +{project_plugins, [erlfmt, {rebar3_ex_doc, "0.2.30"}]}. % Let erlfmt write files on format as opposed to not doing anything {erlfmt, [write]}. From 896a9743b5e0d1423a39cee80997e908218fd8d3 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 18:39:37 +0530 Subject: [PATCH 02/18] Migrate arrow_array --- src/arrow_array.erl | 192 ++++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 112 deletions(-) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index 9c2045b..194ee80 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -15,87 +15,51 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record, a behaviour, and functions to work with Apache Arrow -%% Arrays. -%% -%% This module as mentioned, provides conveniences for working with Apache Arrow -%% Arrays[1] of different Layouts[2]. Firstly, it provides a record to represent -%% all the Layout. Secondly, it provides a behaviour for different layouts to -%% adhere to common functionality. Lastly, it provides functions to work with -%% Arrays in a common manner. -%% -%% == The Structure of an Array == -%% -%% `arrow''s implementation of an Array has the following fields in its -%% record definition: -%% -%%
    -%%
  1. -%% `layout', of type {@link atom()}, which represents the Layout of the -%% Array. -%%
    2. -%%
  2. -%% `type', of type {@link arrow_type:arrow_type()}, which represents -%% the Logical Type[3] of the Array. -%%
    3. -%%
  3. `len', of type {@link pos_integer()}, which represents the Array's Length[4].
    4. -%%
  4. -%% `element_len', of type {@link pos_integer()} or `undefined', which -%% represents the Length of each element in an Array. Currently it only has -%% an integer value in the Fixed-Size List Layout[5] -%%
    5. -%%
  5. -%% `null_count', of type {@link non_neg_integer()}, which represents the -%% Array's Null Count[4], or the number of undefined values in the Array. -%%
    6. -%%
  6. -%% `validity_bitmap', which is a buffer (`arrow_buffer') or the atom -%% `undefined', which represents the Array's Validity Bitmap[6]. -%%
    7. -%%
  7. -%% `offsets', which is a buffer (`arrow_buffer') which represents the -%% Offsets[7], or the start position of each slot in the data buffer of an -%% Array. -%%
    8. -%%
  8. -%% `data', which is a buffer (`arrow_buffer'), which represents the -%% Array's Value Buffer, whose layout differs based on the Array Layout. -%%
    9. -%%
-%% -%% Certain fields are not required for certain layouts. For example, for the -%% Fixed-Sized Primitive Layout, the offsets field is not required, in which -%% case it is assigned as `undefined'. Similarly, the `validity_bitmap' is not -%% required if there are no null values, in which case it is also assigned as -%% `undefined'. -%% -%% == The Behaviour of an Array == -%% -%% As of right now, a layout needs to implement the `c:from_erlang/2' callback. This is -%% then used in the `from_erlang/3' function to create new arrays. -%% -%% == Functions for working with Arrays == -%% -%% As of right now, only functions to access the various fields, to create -%% new arrays, and to serialize arrays to arrow exist. -%% -%% == References == -%% -%% [1]: [https://arrow.apache.org/docs/format/Glossary.html#term-array] -%% -%% [2]: [https://arrow.apache.org/docs/format/Glossary.html#term-physical-layout] -%% -%% [3]: [https://arrow.apache.org/docs/format/Glossary.html#term-type] -%% -%% [4]: [https://arrow.apache.org/docs/format/Columnar.html#null-count] -%% -%% [5]: [https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout] -%% -%% [6]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% -%% [7]: [https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout] -%% @end -module(arrow_array). +-moduledoc """ +Provides a record, a behaviour, and functions to work with Apache Arrow Arrays. + +This module as mentioned, provides conveniences for working with Apache Arrow +[Arrays](https://arrow.apache.org/docs/format/Glossary.html#term-array) of +different +[Layouts](https://arrow.apache.org/docs/format/Glossary.html#term-physical-layout). +Firstly, it provides a record to represent all the Layout. Secondly, it provides +a behaviour for different layouts to adhere to common functionality. Lastly, it +provides functions to work with Arrays in a common manner. + +# The Structure of an Array + +`arrow`'s implementation of an Array has the following fields in its record +definition, each corresponding to a field in the Arrow Columnar Format +specification: + +| Field | Type | Representing | +|-------------------|-----------------------------|-------------------------------------------------------------------------------------------| +| `layout` | `t:atom/0` | Layout | +| `type` | `t:arrow_type:arrow_type/0` | [Logical Type](https://arrow.apache.org/docs/format/Glossary.html#term-type) | +| `len` | `t:pos_integer/0` | [Length](https://arrow.apache.org/docs/format/Columnar.html#array-lengths) | +| `element_len` | `t:pos_integer/0` | Length of each element | +| `null_count` | `t:non_neg_integer/0` | [Null Count](https://arrow.apache.org/docs/format/Columnar.html#null-count) | +| `validity_bitmap` | `arrow_buffer` | [Validity Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) | +| `offsets` | `arrow_buffer` | [Offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) | +| `data` | `arrow_buffer` | Value Buffer | + + +Certain fields are not required for certain layouts. For example, for the +Fixed-Sized Primitive Layout, the offsets field is not required, in which case +it is assigned as `undefined`. Similarly, the `validity_bitmap` is not required +if there are no null values, in which case it is also assigned as `undefined`. + +# The Behaviour of an Array + +As of right now, a layout needs to implement the `c:from_erlang/2` callback. +This is then used in the `from_erlang/3` function to create new arrays. + +# Functions for working with Arrays + +As of right now, only functions to access the various fields, to create +new arrays, and to serialize arrays to arrow exist. +""". -export([ from_erlang/3, layout/1, @@ -113,23 +77,27 @@ -export_type([array/0, layout/0]). +-doc "Represents an Array.". -type array() :: #array{}. -%% Represents an Array. +-doc "Represents the Layout of an Array.". -type layout() :: fixed_primitive | variable_binary | fixed_list | variable_list. -%% Represents the Layout of an Array. %%%%%%%%%%%%%%%%%%%% %% Array Creation %% %%%%%%%%%%%%%%%%%%%% +-doc """ +Creates a new array of a certain layout, given its value and options from its +erlang representation. +""". -callback from_erlang(Value :: [arrow_type:native_type()], Opts :: map()) -> Array :: #array{}. -%% Creates a new array of a certain layout, given its value and options from its -%% erlang representation. -%% @doc A common way to create a new array, given its layout, value, and options. -%% from its erlang representation. +-doc """ +A common way to create a new array, given its layout, value, and options. +from its erlang representation. +""". -spec from_erlang( Layout :: layout(), Value :: [arrow_type:native_type()], @@ -148,42 +116,42 @@ from_erlang(Layout, Value, Opts) -> %% Array Data and Metadata Access %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the layout of an array. +-doc "Returns the layout of an array.". -spec layout(Array :: #array{}) -> Layout :: layout(). layout(Array) -> Array#array.layout. -%% @doc Returns the type of an array. +-doc "Returns the type of an array.". -spec type(Array :: #array{}) -> Type :: arrow_type:arrow_type(). type(Array) -> Array#array.type. -%% @doc Returns the length of an array. +-doc "Returns the length of an array.". -spec len(Array :: #array{}) -> Length :: pos_integer(). len(Array) -> Array#array.len. -%% @doc Returns the length of an array. +-doc "Returns the length of an array.". -spec element_len(Array :: #array{}) -> Length :: pos_integer() | undefined. element_len(Array) -> Array#array.element_len. -%% @doc Returns the null count of an array. +-doc "Returns the null count of an array.". -spec null_count(Array :: #array{}) -> NullCount :: non_neg_integer(). null_count(Array) -> Array#array.null_count. -%% @doc Returns the validity bitmap of an array. +-doc "Returns the validity bitmap of an array.". -spec validity_bitmap(Array :: #array{}) -> ValidityBitmap :: #buffer{} | undefined. validity_bitmap(Array) -> Array#array.validity_bitmap. -%% @doc Returns the offsets of an array. +-doc "Returns the offsets of an array.". -spec offsets(Array :: #array{}) -> Offsets :: #buffer{} | undefined. offsets(Array) -> Array#array.offsets. -%% @doc Returns the data of an array. +-doc "Returns the data of an array.". -spec data(Array :: #array{}) -> Data :: #buffer{} | #array{} | undefined. data(Array) -> Array#array.data. @@ -192,24 +160,24 @@ data(Array) -> %% Array Serialization %% %%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Serializes an array into the Arrow binary form. -%% -%% Serializes the buffers of an Array and concatenates them in the following -%% order: -%% -%%
    -%%
  1. `validity'
    2. -%%
  2. `offsets'
    3. -%%
  3. `data'
    4. -%%
-%% -%% In case an array doesn't have any of the following buffers, it is ommitted. -%% (e.g. validity in arrays with a null count of 0, offsets in fixed primitive -%% arrays). In the case of a nested array, `data' will be serialized form of -%% nested array. -%% -%% Do note that this is just binary form that includes the buffers in an Array, -%% and not IPC. +-doc """ +Serializes an array into the Arrow binary form. + +Serializes the buffers of an Array and concatenates them in the following +order: + +1. `validity` +2. `offsets` +3. `data` + +In case an array doesn't have any of the following buffers, it is ommitted. +(e.g. validity in arrays with a null count of 0, offsets in fixed primitive +arrays). In the case of a nested array, `data` will be serialized form of nested +array. + +Do note that this is just binary form that includes the buffers in an Array, +and not IPC. +""". -spec to_arrow(Array :: #array{}) -> Arrow :: binary(). to_arrow(Array) -> Validity = some(validity_bitmap(Array)), From 1b907e58ed83313b4e3b7d293af2e079772c8bd8 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 21:08:45 +0530 Subject: [PATCH 03/18] Fix erlfmt errors --- src/arrow_array.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index 194ee80..a4eba71 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -16,6 +16,9 @@ % under the License. -module(arrow_array). + +%% TODO Check formatting with erlfmt +%% erlfmt:ignore -moduledoc """ Provides a record, a behaviour, and functions to work with Apache Arrow Arrays. @@ -160,6 +163,8 @@ data(Array) -> %% Array Serialization %% %%%%%%%%%%%%%%%%%%%%%%%%% +%% TODO Check formatting with erlfmt +%% erlfmt:ignore -doc """ Serializes an array into the Arrow binary form. From b4d3301436111ede52ac6ac530cf6178139d1910 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 21:14:48 +0530 Subject: [PATCH 04/18] Quote OTP and Rebar3 versions --- .github/workflows/docs.yml | 4 ++-- .github/workflows/erlang.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ce0d597..7e935bc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,8 +51,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 29.0 - rebar3-version: 3.27.0 + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml index b3db799..99132a9 100644 --- a/.github/workflows/erlang.yml +++ b/.github/workflows/erlang.yml @@ -43,8 +43,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 29.0 - rebar3-version: 3.27.0 + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -81,8 +81,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 29.0 - rebar3-version: 3.27.0 + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -122,8 +122,8 @@ jobs: - name: Install Erlang/OTP uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - otp-version: 29.0 - rebar3-version: 3.27.0 + otp-version: '29.0' + rebar3-version: '3.27.0' - name: Cache Hex packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 From 47c2d54404bb81e1c6844aa04e3e1e28ac436477 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Tue, 14 Jul 2026 21:38:18 +0530 Subject: [PATCH 05/18] Simplify type signatures --- src/arrow_array.erl | 30 +++++++++++++++--------------- src/arrow_array.hrl | 6 +++--- src/arrow_bitmap.erl | 4 ++-- src/arrow_buffer.erl | 15 ++++++++++----- src/arrow_ipc_record_batch.erl | 2 +- src/arrow_offsets.erl | 4 ++-- 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index a4eba71..e04ae61 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -43,9 +43,9 @@ specification: | `len` | `t:pos_integer/0` | [Length](https://arrow.apache.org/docs/format/Columnar.html#array-lengths) | | `element_len` | `t:pos_integer/0` | Length of each element | | `null_count` | `t:non_neg_integer/0` | [Null Count](https://arrow.apache.org/docs/format/Columnar.html#null-count) | -| `validity_bitmap` | `arrow_buffer` | [Validity Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) | -| `offsets` | `arrow_buffer` | [Offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) | -| `data` | `arrow_buffer` | Value Buffer | +| `validity_bitmap` | `t:arrow_buffer:buffer/0` | [Validity Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) | +| `offsets` | `t:arrow_buffer:buffer/0` | [Offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) | +| `data` | `t:arrow_buffer:buffer/0` | Value Buffer | Certain fields are not required for certain layouts. For example, for the @@ -95,7 +95,7 @@ Creates a new array of a certain layout, given its value and options from its erlang representation. """. -callback from_erlang(Value :: [arrow_type:native_type()], Opts :: map()) -> - Array :: #array{}. + Array :: array(). -doc """ A common way to create a new array, given its layout, value, and options. @@ -106,7 +106,7 @@ from its erlang representation. Value :: [arrow_type:native_type()], Opts :: map() | arrow_type:arrow_type() ) -> - Array :: #array{}. + Array :: array(). from_erlang(Layout, Value, Opts) -> case Layout of fixed_primitive -> arrow_fixed_primitive_array:from_erlang(Value, Opts); @@ -120,42 +120,42 @@ from_erlang(Layout, Value, Opts) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -doc "Returns the layout of an array.". --spec layout(Array :: #array{}) -> Layout :: layout(). +-spec layout(Array :: array()) -> Layout :: layout(). layout(Array) -> Array#array.layout. -doc "Returns the type of an array.". --spec type(Array :: #array{}) -> Type :: arrow_type:arrow_type(). +-spec type(Array :: array()) -> Type :: arrow_type:arrow_type(). type(Array) -> Array#array.type. -doc "Returns the length of an array.". --spec len(Array :: #array{}) -> Length :: pos_integer(). +-spec len(Array :: array()) -> Length :: pos_integer(). len(Array) -> Array#array.len. -doc "Returns the length of an array.". --spec element_len(Array :: #array{}) -> Length :: pos_integer() | undefined. +-spec element_len(Array :: array()) -> Length :: pos_integer() | undefined. element_len(Array) -> Array#array.element_len. -doc "Returns the null count of an array.". --spec null_count(Array :: #array{}) -> NullCount :: non_neg_integer(). +-spec null_count(Array :: array()) -> NullCount :: non_neg_integer(). null_count(Array) -> Array#array.null_count. -doc "Returns the validity bitmap of an array.". --spec validity_bitmap(Array :: #array{}) -> ValidityBitmap :: #buffer{} | undefined. +-spec validity_bitmap(Array :: array()) -> ValidityBitmap :: arrow_buffer:buffer() | undefined. validity_bitmap(Array) -> Array#array.validity_bitmap. -doc "Returns the offsets of an array.". --spec offsets(Array :: #array{}) -> Offsets :: #buffer{} | undefined. +-spec offsets(Array :: array()) -> Offsets :: arrow_buffer:buffer() | undefined. offsets(Array) -> Array#array.offsets. -doc "Returns the data of an array.". --spec data(Array :: #array{}) -> Data :: #buffer{} | #array{} | undefined. +-spec data(Array :: array()) -> Data :: arrow_buffer:buffer() | array() | undefined. data(Array) -> Array#array.data. @@ -183,7 +183,7 @@ array. Do note that this is just binary form that includes the buffers in an Array, and not IPC. """. --spec to_arrow(Array :: #array{}) -> Arrow :: binary(). +-spec to_arrow(Array :: array()) -> Arrow :: binary(). to_arrow(Array) -> Validity = some(validity_bitmap(Array)), Offsets = some(offsets(Array)), @@ -191,7 +191,7 @@ to_arrow(Array) -> <>. --spec some(Value :: #array{} | #buffer{} | undefined) -> Binary :: binary(). +-spec some(Value :: array() | arrow_buffer:buffer() | undefined) -> Binary :: binary(). some(undefined) -> <<>>; some(Buffer) when is_record(Buffer, buffer) -> diff --git a/src/arrow_array.hrl b/src/arrow_array.hrl index dbb5ae9..4446bf1 100644 --- a/src/arrow_array.hrl +++ b/src/arrow_array.hrl @@ -23,7 +23,7 @@ len :: pos_integer(), element_len :: pos_integer() | undefined, null_count :: non_neg_integer(), - validity_bitmap :: #buffer{} | undefined, - offsets :: #buffer{} | undefined, - data :: #buffer{} | #array{} + validity_bitmap :: arrow_buffer:buffer() | undefined, + offsets :: arrow_buffer:buffer() | undefined, + data :: arrow_buffer:buffer() | #array{} }). diff --git a/src/arrow_bitmap.erl b/src/arrow_bitmap.erl index 1747514..973429f 100644 --- a/src/arrow_bitmap.erl +++ b/src/arrow_bitmap.erl @@ -65,7 +65,7 @@ %% @doc Returns the Validity Bitmap along with the Null Count, of %% an Array. -spec validity_bitmap(Value :: [arrow_type:native_type()] | list()) -> - {Bitmap :: #buffer{}, non_neg_integer()}. + {Bitmap :: arrow_buffer:buffer(), non_neg_integer()}. validity_bitmap(Value) -> case (lists:member(undefined, Value)) orelse (lists:member(nil, Value)) of true -> @@ -78,7 +78,7 @@ validity_bitmap(Value) -> Value :: [arrow_type:native_type()], Acc :: binary(), NullCount :: non_neg_integer() -) -> {Bitmap :: #buffer{}, NullCount :: non_neg_integer()}. +) -> {Bitmap :: arrow_buffer:buffer(), NullCount :: non_neg_integer()}. bitmap([X1, X2, X3, X4, X5, X6, X7, X8 | Rest], Acc, NullCount) -> %% By assigning B8 as X1's validity, we are following LSB numbering. B8 = validity(X1), diff --git a/src/arrow_buffer.erl b/src/arrow_buffer.erl index 08ff561..79aa125 100644 --- a/src/arrow_buffer.erl +++ b/src/arrow_buffer.erl @@ -56,6 +56,11 @@ -include("arrow_buffer.hrl"). +-export_type([buffer/0]). + +-doc "Represents an Arrow Buffer.". +-type buffer() :: arrow_buffer:buffer(). + %% @doc Creates a new buffer from a list of Erlang values or binaries, given its %% type %% @end @@ -63,7 +68,7 @@ Value :: [arrow_type:native_type()] | binary(), Type :: arrow_type:arrow_longhand_type() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). from_erlang(Data, Type) -> Len = case Type of @@ -82,7 +87,7 @@ from_erlang(Data, Type) -> Type :: arrow_type:arrow_longhand_type(), DataLen :: pos_integer() | undefined ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). from_erlang(Data, Type, DataLen) -> Len = case Type of @@ -97,7 +102,7 @@ from_erlang(Data, Type, DataLen) -> %% @doc Returns an Arrow buffer binary given a buffer. %% @end --spec to_arrow(Buffer :: #buffer{}) -> binary(). +-spec to_arrow(Buffer :: arrow_buffer:buffer()) -> binary(). to_arrow(Buffer) when is_record(Buffer, buffer) -> Type = Buffer#buffer.type, Bin = @@ -114,14 +119,14 @@ to_arrow(_Buffer) -> %% @doc Returns a list of Erlang values or binaries from a buffer. %% @end --spec to_erlang(Buffer :: #buffer{}) -> [arrow_type:native_type()]. +-spec to_erlang(Buffer :: arrow_buffer:buffer()) -> [arrow_type:native_type()]. to_erlang(Buffer) when is_record(Buffer, buffer) -> Buffer#buffer.data; to_erlang(_Buffer) -> erlang:error(badarg). %% @doc Returns the size of the buffer inclusive of padding in bytes. --spec size(Buffer :: #buffer{}) -> pos_integer(). +-spec size(Buffer :: arrow_buffer:buffer()) -> pos_integer(). size(Buffer) -> Len = Buffer#buffer.length * 8, round((Len + arrow_utils:pad_len(Len)) / 8). diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index 8e7fac8..c69b3d7 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -115,7 +115,7 @@ array_data(Array, CurOffset) -> {Buffers ++ [DataBuffer], NewOffset} end. --spec buffer_data(Buffer :: #buffer{}, CurOffset :: non_neg_integer()) -> +-spec buffer_data(Buffer :: arrow_buffer:buffer(), CurOffset :: non_neg_integer()) -> {BufferData :: buffer(), NewOffset :: pos_integer()}. buffer_data(Buffer, CurOffset) -> { diff --git a/src/arrow_offsets.erl b/src/arrow_offsets.erl index 73461c7..31f4b72 100644 --- a/src/arrow_offsets.erl +++ b/src/arrow_offsets.erl @@ -66,7 +66,7 @@ Value :: [arrow_type:native_type()], Type :: arrow_type:arrow_longhand_type() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). new(Values, Type) -> new(Values, Type, length(Values)). @@ -76,7 +76,7 @@ new(Values, Type) -> Type :: arrow_type:arrow_longhand_type(), Length :: pos_integer() ) -> - Buffer :: #buffer{}. + Buffer :: arrow_buffer:buffer(). new(Values, Type, Len) -> Offsets = offsets(Values, [0], 0, Type), arrow_buffer:from_erlang(Offsets, {s, 32}, Len + 1). From b273e8ed87a346d96ec67ebf1a7e87675c36d721 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Wed, 15 Jul 2026 10:53:43 +0530 Subject: [PATCH 06/18] Migrate arrow_buffer --- src/arrow_buffer.erl | 87 ++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 48 deletions(-) diff --git a/src/arrow_buffer.erl b/src/arrow_buffer.erl index 79aa125..0c6cff8 100644 --- a/src/arrow_buffer.erl +++ b/src/arrow_buffer.erl @@ -15,43 +15,35 @@ % specific language governing permissions and limitations % under the License. -%% @doc Buffer implementation for `arrow'. -%% This module adds suppport for buffers, or Contiguous Memory Regions. -%% -%% There are multiple things to know about buffers[1]: -%% -%%
    -%%
  1. -%% Each value it stores is called an element or a slot[2]. -%%
    2. -%%
  2. -%% Each slot's length (in bytes) is a positive integer. As a result when we -%% say that a slot has a length of 1, we mean that each slot has a length of -%% 1 byte. -%%
    3. -%%
  3. -%% The buffer's length in the metadata refers to the unpadded binary's size in bytes. -%%
    4. -%%
  4. -%% All buffers have a size that is a multiple of 64. If their data's length -%% is not a multiple of 64, it must be padded (in this implementation, by -%% zeros). -%%
    5. -%%
  5. -%% Null values are represented in this implementation by zeros. -%%
    6. -%%
  6. -%% In this implementation buffers can be initialized from raw bytes as data -%% apart datatypes supported by Arrow. This is so that the Validity Bitmap -%% Buffer can be initialized. -%%
    7. -%%
-%% -%% [1]: [https://arrow.apache.org/docs/format/Glossary.html#term-buffer] -%% -%% [2]: [https://arrow.apache.org/docs/format/Glossary.html#term-slot] -%% @end -module(arrow_buffer). + +%% TODO Check formatting with erlfmt +%% erlfmt:ignore +-moduledoc """ +Buffer implementation for `arrow`. + +This module adds suppport for buffers, or Contiguous Memory Regions. There are +multiple things to know about +[buffers](https://arrow.apache.org/docs/format/Glossary.html#term-buffer): + +1. Each value it stores is called an element or a + [slot](https://arrow.apache.org/docs/format/Glossary.html#term-slot). + +2. Each slot's length (in bytes) is a positive integer. As a result when we say + that a slot has a length of 1, we mean that each slot has a length of 1 byte. + +3. The buffer's length in the metadata refers to the unpadded binary's size in + bytes. + +4. All buffers have a size that is a multiple of 64. If their data's length is + not a multiple of 64, it must be padded (in this implementation, by zeros). + +5. Null values are represented in this implementation by zeros. + +6. In this implementation buffers can be initialized from raw bytes as data + apart datatypes supported by Arrow. This is so that the Validity Bitmap + Buffer can be initialized. +""". -export([from_erlang/2, from_erlang/3, to_arrow/1, to_erlang/1, size/1]). -include("arrow_buffer.hrl"). @@ -59,11 +51,11 @@ -export_type([buffer/0]). -doc "Represents an Arrow Buffer.". --type buffer() :: arrow_buffer:buffer(). +-type buffer() :: #buffer{}. -%% @doc Creates a new buffer from a list of Erlang values or binaries, given its -%% type -%% @end +-doc """ +Creates a new buffer from a list of Erlang values or binaries, given its type. +""". -spec from_erlang( Value :: [arrow_type:native_type()] | binary(), Type :: arrow_type:arrow_longhand_type() @@ -79,9 +71,10 @@ from_erlang(Data, Type) -> end, from_erlang(Data, Type, Len). -%% @doc Creates a new buffer from a list of Erlang values or binaries, given its -%% type and length -%% @end +-doc """ +Creates a new buffer from a list of Erlang values or binaries, given its +and length +""". -spec from_erlang( Data :: [arrow_type:native_type()] | binary(), Type :: arrow_type:arrow_longhand_type(), @@ -100,8 +93,7 @@ from_erlang(Data, Type, DataLen) -> end, #buffer{type = Type, length = Len, data = Data}. -%% @doc Returns an Arrow buffer binary given a buffer. -%% @end +-doc "Returns an Arrow buffer binary given a buffer.". -spec to_arrow(Buffer :: arrow_buffer:buffer()) -> binary(). to_arrow(Buffer) when is_record(Buffer, buffer) -> Type = Buffer#buffer.type, @@ -117,15 +109,14 @@ to_arrow(Buffer) when is_record(Buffer, buffer) -> to_arrow(_Buffer) -> erlang:error(badarg). -%% @doc Returns a list of Erlang values or binaries from a buffer. -%% @end +-doc "Returns a list of Erlang values or binaries from a buffer.". -spec to_erlang(Buffer :: arrow_buffer:buffer()) -> [arrow_type:native_type()]. to_erlang(Buffer) when is_record(Buffer, buffer) -> Buffer#buffer.data; to_erlang(_Buffer) -> erlang:error(badarg). -%% @doc Returns the size of the buffer inclusive of padding in bytes. +-doc "Returns the size of the buffer inclusive of padding in bytes.". -spec size(Buffer :: arrow_buffer:buffer()) -> pos_integer(). size(Buffer) -> Len = Buffer#buffer.length * 8, From 20937ac1a88d3a26a5231d3e1b22bacac3c48918 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 15:24:26 +0530 Subject: [PATCH 07/18] Migrate arrow_offsets --- src/arrow_offsets.erl | 88 +++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/src/arrow_offsets.erl b/src/arrow_offsets.erl index 31f4b72..4e1d49b 100644 --- a/src/arrow_offsets.erl +++ b/src/arrow_offsets.erl @@ -15,53 +15,49 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Apache Arrow's Offsets -%% -%% Arrow has a concept of offsets[1] in order to tell the length of a slot[2] or -%% a single element in an array of variable-size elements. This module provides -%% support for generating offsets. -%% -%% There are couple of things to remember about offsets: -%% -%%
    -%%
  1. -%% Each element in the offsets coresponds to the distance in bytes of the -%% coresponding element in the values from the beginning of the buffer. -%% -%% I.E., distance of values[j] from beginning of the buffer = offsets[j]. -%%
    2. -%%
  2. -%% The very last element in the offsets is the length of the buffer, or -%% distance of the end of the last slot from the beginning of the buffer. -%%
    3. -%%
  3. -%% Thus, the offsets is one element longer than the values, or: -%% -%% length(offsets) == length(values) + 1 -%%
    4. -%%
  4. -%% Therefore, in order to find the length of a slot, we subtract the offset -%% the current element from the offset of the next element, or: -%% -%% slot[j] = offsets[j + 1] - offsets[j] -%%
    5. -%%
  5. -%% Null values have an offset of 0 as they take no memory in the buffer. -%% Thus, the previous offset and the current offset are equivalent if the -%% current element is a null. -%%
    6. -%%
-%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#terminology] -%% @end -module(arrow_offsets). --export([new/2, new/3]). +-moduledoc """ +Provides support for Apache Arrow's Offsets + +Arrow has a concept of +[offsets](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) +in order to tell the length of a +[slot](https://arrow.apache.org/docs/format/Columnar.html#terminology) or a +single element in an array of variable-size elements. This module provides +support for generating offsets. + +There are couple of things to remember about offsets: + +Firstly, each element in the offsets coresponds to the distance in bytes of the +coresponding element in the values from the beginning of the buffer. + +I.E., distance of `values[j]` from beginning of the buffer = `offsets[j]`. + +Secondly, the very last element in the offsets is the length of the buffer, or +distance of the end of the last slot from the beginning of the buffer. --include("arrow_buffer.hrl"). +Thus, the offsets is one element longer than the values, or: + +``` +length(offsets) == length(values) + 1 +``` + +Therefore, in order to find the length of a slot, we subtract the offset the +current element from the offset of the next element, or: + +``` +slot[j] = offsets[j + 1] - offsets[j] +``` + +Finally, null values have an offset of 0 as they take no memory in the buffer. +Thus, the previous offset and the current offset are equivalent if the current +element is a null. +""". +-export([new/2, new/3]). -%% @doc Returns the offsets array given some values and their type as a buffer. +-doc """ +Returns the offsets array given some values and their type as a buffer. +""". -spec new( Value :: [arrow_type:native_type()], Type :: arrow_type:arrow_longhand_type() @@ -70,7 +66,9 @@ new(Values, Type) -> new(Values, Type, length(Values)). -%% @doc Returns the offsets array given some values, their type and length as a buffer. +-doc """ +Returns the offsets array given some values, their type and length as a buffer. +""". -spec new( Value :: [arrow_type:native_type()], Type :: arrow_type:arrow_longhand_type(), From a59967cec898165b0f8be99425ad6a15b29dabcc Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 15:25:55 +0530 Subject: [PATCH 08/18] Migrate arrow_utils --- src/arrow_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arrow_utils.erl b/src/arrow_utils.erl index ff45a22..a32caea 100644 --- a/src/arrow_utils.erl +++ b/src/arrow_utils.erl @@ -15,8 +15,8 @@ % specific language governing permissions and limitations % under the License. -%% @private Utils module -module(arrow_utils). +-moduledoc false. -export([nesting/1, flatten/1, flatten/3, pad_len/1, pad_len/2]). %% Finds the nesting level of deep list From 3160a51646c1aa1be1c39f9791067226dc6271b1 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 15:35:27 +0530 Subject: [PATCH 09/18] Migrate arrow_bitmap --- src/arrow_bitmap.erl | 76 ++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/src/arrow_bitmap.erl b/src/arrow_bitmap.erl index 973429f..449a9df 100644 --- a/src/arrow_bitmap.erl +++ b/src/arrow_bitmap.erl @@ -15,55 +15,47 @@ % specific language governing permissions and limitations % under the License. -%% @doc Validity Bitmap implementation for `arrow'. -%% -%% Defines a function `validity_bitmap/1' to return the Validity Bitmap[1] along -%% with the Null Count[2], of an Array. -%% -%% An important thing to consider about our implementation of the Null Count is -%% that we need to support both `undefined' and `nil' as null values as they are -%% the conventions for null values in Erlang, and Elixir respectively. -%% -%% There are 5 important characteristics to remember about the validity bitmap: -%% -%%
    -%%
  1. -%% A null value is represented by a 0 bit, and a non null value by a 1 bit. -%%
    2. -%%
  2. -%% Every 8 elements's validities are batched into a byte, which are then -%% reversed as Arrow uses least-significant bit (LSB) numbering (more in -%% attached reference). -%%
    3. -%%
  3. -%% If a "batch" consists of less than 8 elements, its validity needs to be -%% padded by 0 bits so that it can make a byte. -%%
    4. -%%
  4. -%% Each byte is stored in a slot of a Buffer (see docs for -%% `arrow_buffer'). This buffer with the validities of each batch of 8 -%% elements make up what is called the Validity Bitmap. -%%
    5. -%%
  5. -%% If the Null Count is 0, we can allocate the Validity Bitmap as a NULL -%% pointer (which in Erlang's case is `undefined'). -%%
    6. -%%
-%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#null-count] -%% @end -module(arrow_bitmap). --export([validity_bitmap/1]). +-moduledoc """ +Validity Bitmap implementation for `arrow`. + +Defines a function `validity_bitmap/1` to return the [Validity +Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps) +along with the [Null +Count](https://arrow.apache.org/docs/format/Columnar.html#null-count), of an +Array. + +An important thing to consider about our implementation of the Null Count is +that we need to support both `undefined` and `nil` as null values as they are +the conventions for null values in Erlang, and Elixir respectively. + +There are 5 important characteristics to remember about the validity bitmap: --include("arrow_buffer.hrl"). +1. A null value is represented by a 0 bit, and a non null value by a 1 bit. + +2. Every 8 elements's validities are batched into a byte, which are then + reversed as Arrow uses least-significant bit (LSB) numbering (more in + attached reference). + +3. If a "batch" consists of less than 8 elements, its validity needs to be + padded by 0 bits so that it can make a byte. + +4. Each byte is stored in a slot of a Buffer (see docs for `arrow_buffer`). This + buffer with the validities of each batch of 8 elements make up what is called + the Validity Bitmap. + +5. If the Null Count is 0, we can allocate the Validity Bitmap as a NULL pointer + (which in Erlang's case is `undefined`). +""". +-export([validity_bitmap/1]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Validity Bitmap & Null Count %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the Validity Bitmap along with the Null Count, of -%% an Array. +-doc """ +Returns the Validity Bitmap along with the Null Count, of an Array. +""". -spec validity_bitmap(Value :: [arrow_type:native_type()] | list()) -> {Bitmap :: arrow_buffer:buffer(), non_neg_integer()}. validity_bitmap(Value) -> From 4ab8f1b7b387b86d72b5237d174b5e8e4218e2c7 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 18:22:30 +0530 Subject: [PATCH 10/18] Migrate arrow_type --- src/arrow_type.erl | 211 ++++++++++++++++++++------------------------- 1 file changed, 93 insertions(+), 118 deletions(-) diff --git a/src/arrow_type.erl b/src/arrow_type.erl index f7a92b0..a239b01 100644 --- a/src/arrow_type.erl +++ b/src/arrow_type.erl @@ -15,123 +15,94 @@ % specific language governing permissions and limitations % under the License. -%%%----------------------------------------------------------------------------- -%% @doc Provides representation for as well as functions for working with Arrow -%% specific datatypes. -%% -%% This module provides a longhand (a 2 tuple) as well as a shorthand (an atom) -%% represenations for representing a primitive types. Additionally it represents -%% Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types -%% that are supported by this package). -%% -%% Generally, when we say an array A has type X, we mean that each element in A -%% has type X. Thus, an array of type of `Int8' would look like `[1, 2, 3]', but -%% an array of type `List' would look like `[[1, 2], [3]]'. -%% -%% == Primitive Types == -%% -%% A Primitive Type is any type that refers to a single value (as opposed to a -%% list of values). Currently it includes Booleans, Signed Integers, Unsigned -%% Integers, Floats and Binaries. Both a longhand and shorthand syntax had been -%% provided to initiate a type. -%% -%% The following is a comprehensive list of all supported primitive types: -%% -%%
    -%%
  1. Booleans
    2. -%%
  2. Signed Integers: Includes `Int 8', `Int 16', `Int 32' and `Int 64'
    3. -%%
  3. -%% Unsigned Integers: Includes `UInt 8', `UInt 16', `UInt 32' and `UInt 64' -%%
    4. -%%
  4. Floats: Includes `Float 16', `Float 32' and `Float 64'
    5. -%%
  5. Binaries
    6. -%%
-%% -%% === Shorthand Syntax === -%% -%% In the shorthand syntax, you can just use a single atom to refer to a type, -%% such that the atom is `'. The name is a single letter that refers -%% to the type, and the size is that type's size in bits. Take the example of -%% `Int 8'. Its name is `s', and size is `8'. Thus in shorthand, it is `s8'. In -%% the case of datatypes without variations, such as Booleans and Binaries, they -%% are represented by just their datatype. Do note that internally the longhand -%% syntax is used. -%% -%% Here you can find the name of each type: -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%% -%%
TypeNameExample Atom
BooleansN/A`bool'
Signed Integer`s'`s64'
Unsigned Integer`u'`u16'
Float`f'`f32'
BinariesN/A`bin'
-%% -%% Types that are shorthand are under the `t:arrow_shorthand_type()' and -%% `arrow_short_*()' types. -%% -%% === Longhand Syntax === -%% -%% In longhand syntax, you use a 2 tuple to to refer to the type, such that the -%% tuple is `{Name, Size}'. Thus, `Int 8' would be represented as `{s, 8}'. Do -%% note that in the case or Booleans and Binaries their `Size' is `undefined', as -%% the have variable or undefined size. -%% -%% Types that are longhand are under the `t:arrow_longhand_type()' and -%% `arrow_long_*()' types. -%% -%% == Nested Type == -%% -%% A Nested Type is any data structure that supports nesting. This can include -%% Lists, FixedLists, Maps, Structs and more, but is curently limited to -%% FixedLists. Nested Types are represented using a 3 tuple: `{NestedType, Type, -%% Length}'. `NestedType' refers to the data structure that is nesting your -%% primitive type. `Type' refers to the type that is being nested in the data -%% structure. Do note that `Type' can be a Primitive Type, or another Nested -%% Type. Both the shorthand and the longhand syntax can be used or Primitive -%% Types.`Length' refers to the length of each element, and will only be -%% required to represent FixedLists. -%% -%% It is important to remember while nesting that the type refers to the type of -%% each element in an array. A list of `Int8' is 2 dimensional, and a list of -%% `List' is 3 dimensional. -%% -%% == Native Type == -%% -%% A Native Type is any Erlang Datatype that can be serialized by `arrow', -%% and is represented by the type `t:native_type()'. It currently only includes -%% values that are primitive in nature such as booleans, integers, floats, -%% binaries, and additionally the atoms `undefined' and `nil' to represent a -%% `NULL' value. -%% -%% @end -%%%----------------------------------------------------------------------------- -module(arrow_type). +-moduledoc """ +Provides representation for as well as functions for working with Arrow +specific datatypes. + +This module provides a longhand (a 2 tuple) as well as a shorthand (an atom) +represenations for representing a primitive types. Additionally it represents +Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types +that are supported by this package). + +Generally, when we say an array A has type X, we mean that each element in A +has type X. Thus, an array of type of `Int8` would look like `[1, 2, 3]`, but +an array of type `List` would look like `[[1, 2], [3]]`. + +## Primitive Types + +A Primitive Type is any type that refers to a single value (as opposed to a +list of values). Currently it includes Booleans, Signed Integers, Unsigned +Integers, Floats and Binaries. Both a longhand and shorthand syntax had been +provided to initiate a type. + +The following is a comprehensive list of all supported primitive types: + +1. Booleans +2. Signed Integers: Includes `Int 8`, `Int 16`, `Int 32` and + `Int 64` +3. Unsigned Integers: Includes `UInt 8`, `UInt 16`, `UInt 32` and + `UInt 64` +4. Floats: Includes `Float 16`, `Float 32` and `Float 64` +5. Binaries + +### Shorthand Syntax + +In the shorthand syntax, you can just use a single atom to refer to a type, +such that the atom is ``. The name is a single letter that refers +to the type, and the size is that type's size in bits. Take the example of +`Int 8`. Its name is `s`, and size is `8`. Thus in shorthand, it is `s8`. In +the case of datatypes without variations, such as Booleans and Binaries, they +are represented by just their datatype. Do note that internally the longhand +syntax is used. + +Here you can find the name of each table: + +| Type | Name | Example Atom | +|------------------|-------|--------------| +| Booleans | N/A | `bool` | +| Signed Integer | `s` | `s64` | +| Unsigned Integer | `u ` | `u16 ` | +| Float | `f ` | `f32 ` | +| Binaries | N/A | `bin ` | + +Types that are shorthand are under the `t:arrow_shorthand_type()` and +`arrow_short_*()` types. + +### Longhand Syntax + +In longhand syntax, you use a 2 tuple to to refer to the type, such that the +tuple is `{Name, Size}`. Thus, `Int 8` would be represented as `{s, 8}`. Do +note that in the case or Booleans and Binaries their `Size` is `undefined`, as +the have variable or undefined size. + +Types that are longhand are under the `t:arrow_longhand_type()` and +`arrow_long_*()` types. + +### Type + +A Nested Type is any data structure that supports nesting. This can include +Lists, FixedLists, Maps, Structs and more, but is curently limited to +FixedLists. Nested Types are represented using a 3 tuple: `{NestedType, Type, +Length}`. `NestedType` refers to the data structure that is nesting your +primitive type. `Type` refers to the type that is being nested in the data +structure. Do note that `Type` can be a Primitive Type, or another Nested +Type. Both the shorthand and the longhand syntax can be used or Primitive +Types.`Length` refers to the length of each element, and will only be +required to represent FixedLists. + +It is important to remember while nesting that the type refers to the type of +each element in an array. A list of `Int8` is 2 dimensional, and a list of +`List` is 3 dimensional. + +## Native Type + +A Native Type is any Erlang Datatype that can be serialized by `arrow`, +and is represented by the type `t:native_type()`. It currently only includes +values that are primitive in nature such as booleans, integers, floats, +binaries, and additionally the atoms `undefined` and `nil` to represent a +`NULL` value. +""". -export([normalize/1, bit_length/1, byte_length/1, serialize/2]). -export_type([ arrow_type/0, @@ -354,7 +325,9 @@ normalize(_Type) -> erlang:error(badarg). -spec bit_length(Type :: arrow_type()) -> Length :: pos_integer() | undefined. -%% @doc Returns the size of the type in bits. +-doc """ +Returns the size of the type in bits. +""". bit_length(Type) when is_atom(Type) -> bit_length(normalize(Type)); bit_length({Type, Size}) when (Type =:= s) orelse (Type =:= u) orelse (Type =:= f) -> @@ -367,7 +340,9 @@ bit_length(Type) when tuple_size(Type) =:= 3 -> erlang:error(badarg). -spec byte_length(Type :: arrow_type()) -> Length :: pos_integer() | undefined. -%% @doc Returns the size of the type in bytes. +-doc """ +Returns the size of the type in bytes. +""". byte_length(Type) when is_atom(Type) -> byte_length(normalize(Type)); byte_length(Type) when tuple_size(Type) =:= 3 -> From 25174ea9297fa3205b54530c5a39b995d6b3e693 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 19:13:15 +0530 Subject: [PATCH 11/18] Migrate arrow_ipc_* --- src/arrow_ipc_field.erl | 76 ++++++++++----------- src/arrow_ipc_file.erl | 25 ++++--- src/arrow_ipc_message.erl | 119 +++++++++++++++++---------------- src/arrow_ipc_record_batch.erl | 69 ++++++++----------- src/arrow_ipc_schema.erl | 71 ++++++++------------ src/arrow_ipc_type.erl | 27 ++++---- 6 files changed, 182 insertions(+), 205 deletions(-) diff --git a/src/arrow_ipc_field.erl b/src/arrow_ipc_field.erl index 4ed6145..f9eb2f0 100644 --- a/src/arrow_ipc_field.erl +++ b/src/arrow_ipc_field.erl @@ -15,61 +15,55 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with individual fields in -%% a Schema -%% -%% A Field[1] represents a single column in a table (which is represented by a -%% Schema[2]). This module provides a record and a function to manage all the -%% metadata required to represent a column. Metadata such as: -%% -%%
    -%%
  1. -%% `name': The Name of the column. This can either be a string or null. -%%
    2. -%%
  2. -%% `nullable': A boolean representing whether a column can have null values. -%% Generally true. -%%
    3. -%%
  3. -%% `type': The Type of the column, and is of type -%% `arrow_ipc_type:ipc_type()' -%%
    4. -%%
  4. -%% `dictionary': A boolean representing if the column is dictionary encoded -%%
    5. -%%
  5. -%% `children': The child fields of a nested datatype -%%
    6. -%%
  6. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    7. -%%
-%% -%% Currently, dictionary encoding and custom metadata are not supported, but -%% they have been added for forwards comapatibility. -%% -%% [1]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492] -%% -%% [2]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530] -%% @end -module(arrow_ipc_field). +-moduledoc """ +Provides a record and functions to deal with individual fields in +a Schema + +A +[Field](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492) +represents a single column in a table (which is represented by a +[Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530). +This module provides a record and a function to manage all the metadata required +to represent a column. Metadata such as: + +1. `name`: The Name of the column. This can either be a string or + null. +2. `nullable`: A boolean representing whether a column can have null + values. Generally true. +3. `type`: The Type of the column, and is of type + `t:arrow_ipc_type:ipc_type/0` +4. `dictionary`: A boolean representing if the column is dictionary + encoded +5. `children`: The child fields of a nested datatype +6. `custom_metadata`: A list of custom metadata in key-value format + +Currently, dictionary encoding and custom metadata are not supported, but +they have been added for forwards comapatibility. +""". -export([from_erlang/1, from_erlang/2, from_erlang/3]). -include("arrow_ipc_schema.hrl"). -%% @doc Creates a field given the type of the column. Assigns the name as -%% `undefined'. +-doc """ +Creates a field given the type of the column. Assigns the name as +`undefined`. +""". -spec from_erlang(Type :: arrow_ipc_type:ipc_type()) -> Field :: #field{}. from_erlang(Type) -> from_erlang(Type, undefined, []). -%% @doc Creates a field given the type and name of the column. +-doc """ +Creates a field given the type and name of the column. +""". -spec from_erlang(Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined) -> Field :: #field{}. from_erlang(Type, Name) -> from_erlang(Type, Name, []). -%% @doc Creates a field given the type, name and children of the column. +-doc """ +Creates a field given the type, name and children of the column. +""". -spec from_erlang( Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined, Children :: [#field{}] ) -> diff --git a/src/arrow_ipc_file.erl b/src/arrow_ipc_file.erl index ee45e28..a75f3dc 100644 --- a/src/arrow_ipc_file.erl +++ b/src/arrow_ipc_file.erl @@ -15,16 +15,15 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides records and functions to deal with the IPC File. -%% -%% The IPC File[1] is an extension of the IPC Stream[2] that supports random -%% access with the help of a footer which contains the offsets of all the -%% messages. -%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format -%% [2]: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format -%% @end -module(arrow_ipc_file). +-moduledoc """ +Provides records and functions to deal with the IPC File. The [IPC +File](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) is an +extension of the [IPC Stream]( +https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format) that +supports random access with the help of a footer which contains the offsets of +all the messages. +""". -export([from_erlang/2, to_ipc/1]). -include("arrow_ipc_message.hrl"). @@ -34,7 +33,9 @@ %% from_erlang/2 %% %%%%%%%%%%%%%%%%%%% -%% @doc Creates a file given a schema message and a list of record batch messages. +-doc """ +Creates a file given a schema message and a list of record batch messages. +""". -spec from_erlang(Schema :: #message{}, RecordBatches :: [#message{}]) -> #file{}. from_erlang(SchemaMsg, RecordBatches) -> SchemaEMF = arrow_ipc_message:to_ipc(SchemaMsg), @@ -68,7 +69,9 @@ blocks(Offset, [H | T], Blocks, EMFs) -> %% to_ipc/1 %% %%%%%%%%%%%%%% -%% @doc Serializes a file into the IPC File Format +-doc """ +Serializes a file into the IPC File Format. +""". -spec to_ipc(File :: #file{}) -> SerializedFile :: binary(). to_ipc(File) -> Footer = arrow_format_nif:serialize_footer(File#file.footer), diff --git a/src/arrow_ipc_message.erl b/src/arrow_ipc_message.erl index 58c3f14..9d5344c 100644 --- a/src/arrow_ipc_message.erl +++ b/src/arrow_ipc_message.erl @@ -15,72 +15,72 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with the Encapsulated Message -%% Format. -%% -%% A Message is a serialized form of a Schema[1] or a RecordBatch[2] (which may -%% be required to read a serialized array) along with some metadata. This module -%% provides a record and a function to manage all the metadata required to -%% represent a message. Metadata such as: -%% -%%
    -%%
  1. -%% `version': The Apache Arrow Format Version. One of v1..v5. Defaults to v5. -%%
    2. -%%
  2. -%% `header': The metadata of the Schema or RecordBatch -%%
    3. -%%
  3. -%% `body_length': The length of the body in bytes -%%
    4. -%%
  4. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    5. -%%
  5. -%% `body': The actual body. Can be undefined (in the case of Schema) or a -%% binary (in the case of Record Batch). -%%
    6. -%%
-%% -%% Currently, changing the version and custom metadata are not supported, but -%% they have been added for forwards comapatibility. -%% -%% This module also provides the `to_ipc/1' function which serializes the -%% message into the Encapsulated Message Format[3]. However, this function gives -%% incomplete output with invalid metadata, because of an unsatisfied -%% dependency on flatbuffers, which is required for serializing the metadata. -%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#schema-message] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message] -%% -%% [3]: [https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format] -%% @end -module(arrow_ipc_message). +-moduledoc """ +Provides a record and functions to deal with the Encapsulated Message +Format. + +A Message is a serialized form of a +[Schema](https://arrow.apache.org/docs/format/Columnar.html#schema-message) or a +[RecordBatch](https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message) +(which may be required to read a serialized array) along with some metadata. +This module provides a record and a function to manage all the metadata required +to represent a message. Metadata such as: + +1. `version`: The Apache Arrow Format Version. One of `v1..v5`. + Defaults to v5. +2. `header`: The metadata of the Schema or RecordBatch +3. `body_length`: The length of the body in bytes +4. `custom_metadata`: A list of custom metadata in key-value format +5. `body`: The actual body. Can be undefined (in the case of Schema) + or a binary (in the case of Record Batch). + +Currently, changing the version and custom metadata are not supported, but they +have been added for forwards comapatibility. + +This module also provides the `to_ipc/1` function which serializes the message +into the [Encapsulated Message +Format](https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format). +However, this function gives *incomplete output* with invalid metadata, because +of an unsatisfied dependency on flatbuffers, which is required for serializing +the metadata. +""". -export([from_erlang/1, from_erlang/2, to_ipc/1, to_stream/1, metadata_len/1, body_from_erlang/1]). -export_type([metadata_version/0, key_value/0]). -include("arrow_ipc_message.hrl"). +-doc """ +The Arrow version. See the +[definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L28-L49) +for more info: +""". -type metadata_version() :: v1 | v2 | v3 | v4 | v5. -%% The Arrow version. See the definition for more info: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L28-L49] +-doc """ +Key-Value structure for custom metadata. See the +[definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L432-L439) +for more info: +""". -type key_value() :: #{key => string(), value => string()}. -%% Key-Value structure for custom metadata. See the definition for more info: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L432-L439] -%% @doc Creates a message given a data header. +-doc """ +Creates a message given a data header. +""". -spec from_erlang(Header :: #schema{} | #record_batch{}) -> Message :: #message{}. from_erlang(Header) -> #message{header = Header, body_length = 0}. -%% @doc Creates a message given a data header and a body. +-doc """ +Creates a message given a data header and a body. +""". -spec from_erlang(Header :: #schema{} | #record_batch{}, Body :: binary()) -> Message :: #message{}. from_erlang(Header, Body) -> #message{header = Header, body = Body, body_length = byte_size(Body)}. -%% @doc Serializes a message into the Encapsulated Message Format. +-doc """ +Serializes a message into the Encapsulated Message Format. +""". -spec to_ipc(Message :: #message{}) -> EMF :: binary(). to_ipc(Message) -> %% 0xFFFFFFFF in int32 @@ -98,7 +98,9 @@ to_ipc(Message) -> <>. -%% @doc Serializes a list of messages or EMFs into a Stream. +-doc """ +Serializes a list of messages or EMFs into a Stream. +""". -spec to_stream(Messages :: [#message{}] | [binary()]) -> Stream :: binary(). to_stream([H | _] = Messages) -> Msgs = @@ -117,7 +119,9 @@ to_stream([H | _] = Messages) -> %% metadata_len/1 %% %%%%%%%%%%%%%%%%%%%% -%% @doc Returns the Metadata Length of an EMF. +-doc """ +Returns the Metadata Length of an EMF. +""". -spec metadata_len(EMF :: binary()) -> MetadataLen :: non_neg_integer(). metadata_len(EMF) -> <<_:32, MetadataLen:32, _Rest/binary>> = EMF, @@ -127,13 +131,14 @@ metadata_len(EMF) -> %% body_from_erlang/1 %% %%%%%%%%%%%%%%%%%%%%%%%% -%% @doc Returns the body of message from a list of arrays. -%% -%% Shorthand for: -%% ``` -%% <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>> -%% ''' -%% @end +-doc """ +Returns the body of message from a list of arrays. + +Shorthand for: +``` +<<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>> +``` +""". -spec body_from_erlang(Columns :: [#array{}]) -> Body :: binary(). body_from_erlang(Columns) -> <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>>. diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index c69b3d7..0d3f3dc 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -15,46 +15,33 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with RecordBatches -%% -%% A RecordBatch[1] represents a list of equal length arrays and their -%% coresponding buffers. This module provides a record and a function to manage -%% all the metadata required to represent a RecordBatch. Metadata such as: -%% -%%
    -%%
  1. -%% `length': The number of rows or records. In other words, the length of -%% an array. -%%
    2. -%%
  2. -%% `nodes': A list of maps, where each map has the length and null count -%% of an array -%%
    3. -%%
  3. -%% `buffers': A list of maps, where each map has the length and the offset -%% (from the beginning of the message body) of a buffer of an array. -%%
    4. -%%
  4. -%% `compression': The compression applied on the body of the Record Batch. -%% Can either by `undefined' (i.e. no compression), `zstd' for Zstandard[2], -%% or `lz4_frame' for LZ4 Frame[3]. Defaults to `undefined'. -%%
    5. -%%
-%% -%% Currently, compression is not supported, but it has been added for forwards -%% comapatibility. -%% -%% You can find RecordBatches in the Arrow spec here[4]. -%% -%% [1]: [https://github.com/apache/arrow/blob/16328f0ccc73b7df665b4a18feb6adf26b7aa0e2/format/Message.fbs#L81-L102] -%% -%% [2]: [https://facebook.github.io/zstd/] -%% -%% [3]: [https://android.googlesource.com/platform/external/lz4/+/HEAD/doc/lz4_Frame_format.md] -%% -%% [4]: [https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message] -%% @end -module(arrow_ipc_record_batch). +-moduledoc """ +Provides a record and functions to deal with RecordBatches +A [RecordBatch](https://github.com/apache/arrow/blob/16328f0ccc73b7df665b4a18feb6adf26b7aa0e2/format/Message.fbs#L81-L102) represents a list of equal length arrays and their +coresponding buffers. This module provides a record and a function to manage +all the metadata required to represent a RecordBatch. Metadata such as: + +1. `length`: The number of rows or records. In other words, the + length of an array. +2. `nodes`: A list of maps, where each map has the length and null + count of an array +3. `buffers`: A list of maps, where each map has the length and the + offset (from the beginning of the message body) of a buffer of an + array. + +4. `compression`: The compression applied on the body of the Record + Batch. Can either by `undefined` (i.e. no compression), `zstd` for + [Zstandard](https://facebook.github.io/zstd/), or `lz4_frame` for [LZ4 + Frame](https://android.googlesource.com/platform/external/lz4/+/HEAD/doc/lz4_Frame_format.md). + Defaults to `undefined`. + +Currently, compression is not supported, but it has been added for forwards +comapatibility. + +You can find RecordBatches in the Arrow spec +[here](https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message). +""". -export([from_erlang/1]). -export_type([field_node/0, buffer/0]). @@ -64,7 +51,9 @@ -type field_node() :: #{length => pos_integer(), null_count => non_neg_integer()}. -type buffer() :: #{offset => non_neg_integer(), length => pos_integer()}. -%% @doc Creates a RecordBatch given a list of arrays +-doc """ +Creates a RecordBatch given a list of arrays +""". -spec from_erlang(Arrays :: [#array{}]) -> RecordBatch :: #record_batch{}. from_erlang(Arrays) -> FieldNodes = lists:map( diff --git a/src/arrow_ipc_schema.erl b/src/arrow_ipc_schema.erl index 22632fd..a26d766 100644 --- a/src/arrow_ipc_schema.erl +++ b/src/arrow_ipc_schema.erl @@ -15,58 +15,45 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides a record and functions to deal with Schemas -%% -%% A Schema[1] represents a table, or a list of arrays of equal length. This -%% module provides a record and a function to manage all the metadata required -%% to represent a schema. Metadata such as: -%% -%%
    -%%
  1. -%% `endianness': The Endianness of the table. One of `little' or `big'. -%% Defaults to `little'. -%%
    2. -%%
  2. -%% `fields': The list of fields[2] in a table. -%%
    3. -%%
  3. -%% `type': The Layout of the column -%%
    4. -%%
  4. -%% `custom_metadata': A list of custom metadata in key-value format -%%
    5. -%%
  5. -%% `features': Any features used by the table which may not be present in -%% other implementations of Arrow. -%%
    6. -%%
-%% -%% Currently, big endianness, custom metadata and features are not supported, -%% but they have been added for forwards comapatibility. -%% -%% You can find Schemas in the Arrow spec here[3]. -%% -%% [1]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530] -%% -%% [2]: [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492] -%% -%% [3]: [https://arrow.apache.org/docs/format/Columnar.html#schema-message] -%% @end -module(arrow_ipc_schema). +-moduledoc """ +Provides a record and functions to deal with Schemas A +[Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530)[1] +represents a table, or a list of arrays of equal length. This module provides a +record and a function to manage all the metadata required to represent a schema. +Metadata such as: + +1. `endianness`: The Endianness of the table. One of `little` or + `big`. Defaults to `little`. +2. `fields`: The list of [fields](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L469-L492) in a table. +3. `type`: The Layout of the column +4. `custom_metadata`: A list of custom metadata in key-value format +5. `features`: Any features used by the table which may not be + present in other implementations of Arrow. + +Currently, big endianness, custom metadata and features are not supported, but +they have been added for forwards comapatibility. + +You can find Schemas in the Arrow spec +[here](https://arrow.apache.org/docs/format/Columnar.html#schema-message). +""". -export([from_erlang/1]). -export_type([endianness/0, feature/0]). -include("arrow_ipc_schema.hrl"). +-doc "Endianness of the data. Either `little` or `big`.". -type endianness() :: little | big. -%% Endianness of the data. Either `little' or `big'. +-doc """ +Features used in the data which may not be present in other implementations. +See the [definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L51-L78). +""". -type feature() :: unused | dictionary_replacement | compressed_body. -%% Features used in the data which may not be present in other implementations. -%% See the definition: -%% [https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L51-L78] -%% @doc Creates a Schema given an ordered list of fields. +-doc """ +Creates a Schema given an ordered list of fields. +""". -spec from_erlang(Fields :: [#field{}]) -> Schema :: #schema{}. from_erlang(Fields) -> #schema{fields = Fields}. diff --git a/src/arrow_ipc_type.erl b/src/arrow_ipc_type.erl index f4b1e7c..5e4e172 100644 --- a/src/arrow_ipc_type.erl +++ b/src/arrow_ipc_type.erl @@ -15,20 +15,17 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides types and functions to work with IPC types. -%% -%% This module provides functions and types to produce the types in IPC Schema -%% definitions[1]. These types are generated according to these[2] definitions. -%% The types have been represented in the form `{TypeName, Metadata}', where -%% `TypeName' is the name of the type and is an atom, and `Metadata' is a map of -%% all the metadata associated with it. In case a type has no metadata -%% associated with it, it is represented as just `TypeName' -%% -%% [1]:https://github.com/apache/arrow/blob/main/format/Schema.fbs -%% -%% [2]: https://github.com/apache/arrow/blob/main/format/Schema.fbs#L82-L430 -%% @end -module(arrow_ipc_type). +-moduledoc """ +Provides types and functions to work with IPC types. This module provides +functions and types to produce the types in IPC Schema +[definitions](https://github.com/apache/arrow/blob/main/format/Schema.fbs). +These types are generated according to `Type` union in the schema. The types +have been represented in the form `{TypeName, Metadata}`, where `TypeName` is +the name of the type and is an atom, and `Metadata` is a map of all the metadata +associated with it. In case a type has no metadata associated with it, it is +represented as just `TypeName` +""". -export([from_erlang/1]). -export_type([ ipc_type/0, @@ -90,7 +87,9 @@ %% from_erlang/1 %% %%%%%%%%%%%%%%%%%%% -%% @doc Returns the IPC Type for an `#array{}'. +-doc """ +Returns the IPC Type for an `t:arrow_array:array()`. +""". -spec from_erlang(Array :: arrow_array:array()) -> Type :: ipc_type(). from_erlang(Array) -> case Array#array.layout of From 36ad36d4cc77ed21b09871d87e39e0a6a187af7f Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 19:37:14 +0530 Subject: [PATCH 12/18] Simplify arrow_ipc_* type signatures --- src/arrow_ipc_field.erl | 14 ++++++++++---- src/arrow_ipc_file.erl | 10 +++++++--- src/arrow_ipc_message.erl | 18 ++++++++++++------ src/arrow_ipc_record_batch.erl | 7 ++++--- src/arrow_ipc_schema.erl | 7 +++++-- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/arrow_ipc_field.erl b/src/arrow_ipc_field.erl index f9eb2f0..1b6938a 100644 --- a/src/arrow_ipc_field.erl +++ b/src/arrow_ipc_field.erl @@ -42,14 +42,20 @@ Currently, dictionary encoding and custom metadata are not supported, but they have been added for forwards comapatibility. """. -export([from_erlang/1, from_erlang/2, from_erlang/3]). +-export_type([field/0]). -include("arrow_ipc_schema.hrl"). +-doc """ +Represents an IPC Field. +""". +-type field() :: #field{}. + -doc """ Creates a field given the type of the column. Assigns the name as `undefined`. """. --spec from_erlang(Type :: arrow_ipc_type:ipc_type()) -> Field :: #field{}. +-spec from_erlang(Type :: arrow_ipc_type:ipc_type()) -> Field :: field(). from_erlang(Type) -> from_erlang(Type, undefined, []). @@ -57,7 +63,7 @@ from_erlang(Type) -> Creates a field given the type and name of the column. """. -spec from_erlang(Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined) -> - Field :: #field{}. + Field :: field(). from_erlang(Type, Name) -> from_erlang(Type, Name, []). @@ -65,8 +71,8 @@ from_erlang(Type, Name) -> Creates a field given the type, name and children of the column. """. -spec from_erlang( - Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined, Children :: [#field{}] + Type :: arrow_ipc_type:ipc_type(), Name :: string() | undefined, Children :: [field()] ) -> - Field :: #field{}. + Field :: field(). from_erlang(Type, Name, Children) -> #field{name = Name, type = Type, children = Children}. diff --git a/src/arrow_ipc_file.erl b/src/arrow_ipc_file.erl index a75f3dc..f3eb82c 100644 --- a/src/arrow_ipc_file.erl +++ b/src/arrow_ipc_file.erl @@ -29,6 +29,8 @@ all the messages. -include("arrow_ipc_message.hrl"). -include("arrow_ipc_file.hrl"). +-type file() :: #file{}. + %%%%%%%%%%%%%%%%%%% %% from_erlang/2 %% %%%%%%%%%%%%%%%%%%% @@ -36,7 +38,9 @@ all the messages. -doc """ Creates a file given a schema message and a list of record batch messages. """. --spec from_erlang(Schema :: #message{}, RecordBatches :: [#message{}]) -> #file{}. +-spec from_erlang( + Schema :: arrow_ipc_message:message(), RecordBatches :: [arrow_ipc_message:message()] +) -> file(). from_erlang(SchemaMsg, RecordBatches) -> SchemaEMF = arrow_ipc_message:to_ipc(SchemaMsg), {RecordBatchBlocks, EMFs} = blocks(byte_size(SchemaEMF), RecordBatches, [], []), @@ -50,7 +54,7 @@ from_erlang(SchemaMsg, RecordBatches) -> %% Returns the blocks as wells as the intermediate EMFs used to generate each block. -spec blocks( Offset :: non_neg_integer(), - Messages :: [#message{}], + Messages :: [arrow_ipc_message:message()], Blocks :: [#block{}], EMFs :: [binary()] ) -> {Blocks :: [#block{}], EMFs :: [binary()]}. @@ -72,7 +76,7 @@ blocks(Offset, [H | T], Blocks, EMFs) -> -doc """ Serializes a file into the IPC File Format. """. --spec to_ipc(File :: #file{}) -> SerializedFile :: binary(). +-spec to_ipc(File :: file()) -> SerializedFile :: binary(). to_ipc(File) -> Footer = arrow_format_nif:serialize_footer(File#file.footer), Sz = byte_size(Footer), diff --git a/src/arrow_ipc_message.erl b/src/arrow_ipc_message.erl index 9d5344c..18265e1 100644 --- a/src/arrow_ipc_message.erl +++ b/src/arrow_ipc_message.erl @@ -46,10 +46,13 @@ of an unsatisfied dependency on flatbuffers, which is required for serializing the metadata. """. -export([from_erlang/1, from_erlang/2, to_ipc/1, to_stream/1, metadata_len/1, body_from_erlang/1]). --export_type([metadata_version/0, key_value/0]). +-export_type([message/0, metadata_version/0, key_value/0]). -include("arrow_ipc_message.hrl"). +-doc "Represents a Message.". +-type message() :: #message{}. + -doc """ The Arrow version. See the [definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L28-L49) @@ -67,21 +70,24 @@ for more info: -doc """ Creates a message given a data header. """. --spec from_erlang(Header :: #schema{} | #record_batch{}) -> Message :: #message{}. +-spec from_erlang(Header :: arrow_ipc_schema:schema() | arrow_ipc_record_batch:record_batch()) -> + Message :: message(). from_erlang(Header) -> #message{header = Header, body_length = 0}. -doc """ Creates a message given a data header and a body. """. --spec from_erlang(Header :: #schema{} | #record_batch{}, Body :: binary()) -> Message :: #message{}. +-spec from_erlang( + Header :: arrow_ipc_schema:schema() | arrow_ipc_record_batch:record_batch(), Body :: binary() +) -> Message :: message(). from_erlang(Header, Body) -> #message{header = Header, body = Body, body_length = byte_size(Body)}. -doc """ Serializes a message into the Encapsulated Message Format. """. --spec to_ipc(Message :: #message{}) -> EMF :: binary(). +-spec to_ipc(Message :: message()) -> EMF :: binary(). to_ipc(Message) -> %% 0xFFFFFFFF in int32 Continuation = <<-1:32>>, @@ -101,7 +107,7 @@ to_ipc(Message) -> -doc """ Serializes a list of messages or EMFs into a Stream. """. --spec to_stream(Messages :: [#message{}] | [binary()]) -> Stream :: binary(). +-spec to_stream(Messages :: [message()] | [binary()]) -> Stream :: binary(). to_stream([H | _] = Messages) -> Msgs = if @@ -139,6 +145,6 @@ Shorthand for: <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>> ``` """. --spec body_from_erlang(Columns :: [#array{}]) -> Body :: binary(). +-spec body_from_erlang(Columns :: [arrow_array:array()]) -> Body :: binary(). body_from_erlang(Columns) -> <<<<(arrow_array:to_arrow(Array))/binary>> || Array <- Columns>>. diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index 0d3f3dc..0a3d517 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -43,18 +43,19 @@ You can find RecordBatches in the Arrow spec [here](https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message). """. -export([from_erlang/1]). --export_type([field_node/0, buffer/0]). +-export_type([field_node/0, buffer/0, record_batch/0]). -include("arrow_ipc_record_batch.hrl"). -include("arrow_array.hrl"). -type field_node() :: #{length => pos_integer(), null_count => non_neg_integer()}. -type buffer() :: #{offset => non_neg_integer(), length => pos_integer()}. +-type record_batch() :: #record_batch{}. -doc """ Creates a RecordBatch given a list of arrays """. --spec from_erlang(Arrays :: [#array{}]) -> RecordBatch :: #record_batch{}. +-spec from_erlang(Arrays :: [arrow_array:array()]) -> RecordBatch :: record_batch(). from_erlang(Arrays) -> FieldNodes = lists:map( fun(X) -> #{length => X#array.len, null_count => X#array.null_count} end, Arrays @@ -72,7 +73,7 @@ from_erlang(Arrays) -> #record_batch{length = Length, nodes = FieldNodes, buffers = lists:reverse(Buffers)}. %% Returns the buffer data for all the buffers and nested arrays in an array. --spec array_data(Array :: #array{}, CurOffset :: non_neg_integer()) -> +-spec array_data(Array :: arrow_array:array(), CurOffset :: non_neg_integer()) -> {[buffer()], NewOffset :: non_neg_integer()}. array_data(Array, CurOffset) -> Bitmap = Array#array.validity_bitmap, diff --git a/src/arrow_ipc_schema.erl b/src/arrow_ipc_schema.erl index a26d766..3e56f39 100644 --- a/src/arrow_ipc_schema.erl +++ b/src/arrow_ipc_schema.erl @@ -38,7 +38,7 @@ You can find Schemas in the Arrow spec [here](https://arrow.apache.org/docs/format/Columnar.html#schema-message). """. -export([from_erlang/1]). --export_type([endianness/0, feature/0]). +-export_type([endianness/0, feature/0, schema/0]). -include("arrow_ipc_schema.hrl"). @@ -51,9 +51,12 @@ See the [definition](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569 """. -type feature() :: unused | dictionary_replacement | compressed_body. +-doc "Represents a schema". +-type schema() :: #schema{}. + -doc """ Creates a Schema given an ordered list of fields. """. --spec from_erlang(Fields :: [#field{}]) -> Schema :: #schema{}. +-spec from_erlang(Fields :: [arrow_ipc_field:field()]) -> Schema :: schema(). from_erlang(Fields) -> #schema{fields = Fields}. From ec66c411312a247ff9d2094c8959c8b4809870be Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 20:00:27 +0530 Subject: [PATCH 13/18] Migrate arrow array modules --- src/arrow_fixed_list_array.erl | 68 +++++++++++++--------------- src/arrow_fixed_primitive_array.erl | 69 +++++++++++++---------------- src/arrow_variable_binary_array.erl | 28 +++++++----- src/arrow_variable_list_array.erl | 47 ++++++++++---------- 4 files changed, 103 insertions(+), 109 deletions(-) diff --git a/src/arrow_fixed_list_array.erl b/src/arrow_fixed_list_array.erl index 53a0d49..bf2f238 100644 --- a/src/arrow_fixed_list_array.erl +++ b/src/arrow_fixed_list_array.erl @@ -15,50 +15,46 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size List Layout. -%% -%% This module provides support for the Fixed-Size List Layout[1], which is an -%% layout that supports storing a list of lists of a specific length and -%% nesting. -%% -%% == Invalid Input == -%% -%% It is important that care is taken when passing input values to this module. -%% For performance reasons, the input is not validated. The function crashes on -%% nesting that is inconsitent: a. with the type, b. between elements. The lists -%% are checked to have the same length as each other at the top level, but not -%% for deeper levels. The behaviour of the module on invalid input -%% CANNOT BE GUARANTEED. Therefore, one must be careful to not -%% to CRASH THE PROCESS or worse still, PRODUCE INVALID -%% OUTPUT. -%% -%% Any input must follow the following rules: -%% -%%
    -%%
  1. The length of each element must be consistent with the type
    2. -%%
  2. The nesting of each element must be consistent with the type
    3. -%%
  3. The length of each element must be consistent with each other
    4. -%%
  4. The nesting of each element must be consistent with each other
    5. -%%
  5. -%% The nested type is a `fixed_list' (as only then can fixed size be -%% guaranteed) -%%
    6. -%%
-%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout -module(arrow_fixed_list_array). +-moduledoc """ +Provides support for Arrow's Fixed-Size List Layout. + +This module provides support for the [Fixed-Size List +Layout](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout), +which is an layout that supports storing a list of lists of a specific length +and nesting. + +## Invalid Input + +It is important that care is taken when passing input values to this module. For +performance reasons, the input is not validated. The function crashes on nesting +that is inconsitent: a. with the type, b. between elements. The lists are +checked to have the same length as each other at the top level, but not for +deeper levels. The behaviour of the module on invalid input **CANNOT BE +GUARANTEED**. Therefore, one must be careful to not to **CRASH THE PROCESS** or +worse still, **PRODUCE INVALID OUTPUT**. + +Any input must follow the following rules: + +1. The length of each element must be consistent with the type +2. The nesting of each element must be consistent with the type +3. The length of each element must be consistent with each other +4. The nesting of each element must be consistent with each other +5. The nested type is a `fixed_list` (as only then can fixed size be + guaranteed) +""". -behaviour(arrow_array). -export([from_erlang/2]). -include("arrow_array.hrl"). -%% @doc Creates a Fixed-Size List Array given the values and type. -%% -%% Accepts a map with the type, or the type directly. -%% @end +-doc """ +Creates a Fixed-Size List Array given the values and type. +Accepts a map with the type, or the type directly. +""". -spec from_erlang(Values :: list(), Type :: map() | arrow_type:arrow_type()) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Values, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined -> diff --git a/src/arrow_fixed_primitive_array.erl b/src/arrow_fixed_primitive_array.erl index 1f5bc5b..d37d31b 100644 --- a/src/arrow_fixed_primitive_array.erl +++ b/src/arrow_fixed_primitive_array.erl @@ -15,40 +15,30 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size Primitive Layout. -%% -%% The Primitive Layout[1] uses the following metadata in the `arrow_array': -%% -%%
    -%%
  1. -%% `layout', of type {@link atom()}, and a constant value of `fixed_primitive'. -%%
    2. -%%
  2. -%% `type', of type `t:arrow_type:arrow_primitive_type()', which -%% represents the Logical Type of the Array. -%%
    3. -%%
  3. `len', of type {@link pos_integer()}, which represents the Array's Length.
    4. -%%
  4. -%% `null_count', of type {@link non_neg_integer()}, which represents the Array's -%% Null Count, or the number of undefined values in the Array. -%%
    5. -%%
  5. -%% `validity_bitmap', which is a buffer (`arrow_buffer') or the atom -%% `undefined', which represents the Array's Validity Bitmap[2]. -%%
    6. -%%
  6. -%% `data', which is a buffer (`arrow_buffer'), which represents the Array's Value Buffer. -%%
    7. -%%
-%% -%% The `offsets' field will be allocated as `undefined'. Same goes for the -%% `validity_bitmap' field in the case of a 0 `null_count'. -%% -%% [1]: [https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout] -%% -%% [2]: [https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps] -%% @end -module(arrow_fixed_primitive_array). +-moduledoc """ +Provides support for Arrow's Fixed-Size Primitive Layout. The [Primitive +Layout](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout) +uses the following metadata in the `arrow_array`: + +1. `layout`, of type `t:atom/0`, and a constant value of + `fixed_primitive`. +2. `type`, of type `t:arrow_type:arrow_primitive_type/0`, which + represents the Logical Type of the Array. +3. `len`, of type `t:pos_integer/0`, which represents the Array's + Length. +4. `null_count`, of type `t:non_neg_integer/0`, which represents + the Array's Null Count, or the number of undefined values in the + Array. +5. `validity_bitmap`, which is a buffer (`arrow_buffer`) or the + atom `undefined`, which represents the Array's [Validity + Bitmap](https://arrow.apache.org/docs/format/Columnar.html#validity-bitmaps). +6. `data`, which is a buffer (`arrow_buffer`), which represents the + Array's Value Buffer. + +The `offsets` field will be allocated as `undefined`. Same goes for the +`validity_bitmap` field in the case of a 0 `null_count`. +""". -behaviour(arrow_array). -export([from_erlang/2]). @@ -59,16 +49,17 @@ %% Array Creation %% %%%%%%%%%%%%%%%%%%%% -%% @doc Creates a new primitive array, given its value and type from its erlang -%% representation. -%% -%% Accepts a map with the type, or the type directly. -%% @end +-doc """ +Creates a new primitive array, given its value and type from its erlang +representation. + +Accepts a map with the type, or the type directly. +""". -spec from_erlang( Value :: [arrow_type:native_type()], Type :: map() | arrow_type:arrow_primitive_type() ) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Value, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined -> diff --git a/src/arrow_variable_binary_array.erl b/src/arrow_variable_binary_array.erl index 574b150..537d76a 100644 --- a/src/arrow_variable_binary_array.erl +++ b/src/arrow_variable_binary_array.erl @@ -15,26 +15,34 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Variable-Sized Binary Layout. -%% -%% The Variable-Sized Binary Layout[1] provides support for storing binaries of -%% varying length in a way similar to the primitive layout, i.e. in a 1 -%% Dimensional Array. -module(arrow_variable_binary_array). +-moduledoc """ +Provides support for Arrow's Variable-Sized Binary Layout. + +The [Variable-Sized Binary +Layout](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout) +provides support for storing binaries of varying length in a way similar to the +primitive layout, i.e. in a 1 Dimensional Array. +""". -behaviour(arrow_array). -export([from_erlang/1, from_erlang/2]). -include("arrow_array.hrl"). -%% @doc Creates a Variable-Sized Binary Array given the values and options in the form of -%% a map, from its erlang representation. --spec from_erlang(Values :: [arrow_type:native_type()], Opts :: map()) -> Array :: #array{}. +-doc """ +Creates a Variable-Sized Binary Array given the values and options in the form of +a map, from its erlang representation. +""". +-spec from_erlang(Values :: [arrow_type:native_type()], Opts :: map()) -> + Array :: arrow_array:array(). from_erlang(Values, _Opts) -> from_erlang(Values). -%% @doc Creates a Variable-Sized Binary Array given the values --spec from_erlang(Values :: [arrow_type:native_type()]) -> Array :: #array{}. +-doc """ +Creates a Variable-Sized Binary Array given the values +""". +-spec from_erlang(Values :: [arrow_type:native_type()]) -> Array :: arrow_array:array(). from_erlang(Values) -> Len = length(Values), {Bitmap, NullCount} = arrow_bitmap:validity_bitmap(Values), diff --git a/src/arrow_variable_list_array.erl b/src/arrow_variable_list_array.erl index 79fc6af..46cdc97 100644 --- a/src/arrow_variable_list_array.erl +++ b/src/arrow_variable_list_array.erl @@ -15,30 +15,29 @@ % specific language governing permissions and limitations % under the License. -%% @doc Provides support for Arrow's Fixed-Size List Layout. -%% -%% This module provides support for the Fixed-Size List Layout[1], which is an -%% layout that supports storing a list of lists of a specific length and -%% nesting. -%% -%% == Invalid Input == -%% -%% It is important that care is taken when passing input values to this module. -%% For performance reasons, the input is not validated. The function crashes on -%% nesting that is inconsitent: a. with the type, b. between elements. The -%% behaviour on invalid input CANNOT BE GUARANTEED. Therefore, -%% one must be careful to not to CRASH THE PROCESS or worse -%% still, PRODUCE INVALID OUTPUT. -%% -%% Any input must follow the following rules: -%% -%%
    -%%
  1. The nesting of each element must be consistent with the type
    2. -%%
  2. The nesting of each element must be consistent with each other
    3. -%%
-%% -%% [1]: https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout -module(arrow_variable_list_array). +-moduledoc """ +Provides support for Arrow's Fixed-Size List Layout. + +This module provides support for the [Fixed-Size List +Layout](https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout), +which is an layout that supports storing a list of lists of a specific length +and nesting. + +## Invalid Input + +It is important that care is taken when passing input values to this +module. For performance reasons, the input is not validated. The +function crashes on nesting that is inconsitent: a. with the type, b. +between elements. The behaviour on invalid input **CANNOT BE +GUARANTEED**. Therefore, one must be careful to not to **CRASH THE +PROCESS** or worse still, **PRODUCE INVALID OUTPUT**. + +Any input must follow the following rules: + +1. The nesting of each element must be consistent with the type +2. The nesting of each element must be consistent with each other +""". -behaviour(arrow_array). @@ -47,7 +46,7 @@ -include("arrow_array.hrl"). -spec from_erlang(Values :: list(), Type :: map() | arrow_type:arrow_type()) -> - Array :: #array{}. + Array :: arrow_array:array(). from_erlang(Values, Opts) when is_map(Opts) -> case maps:get(type, Opts, undefined) of undefined -> From 84822c1f6c400b4d0360261b486378dedfba9c86 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Sun, 19 Jul 2026 20:14:54 +0530 Subject: [PATCH 14/18] Fix element_len docs --- src/arrow_array.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index e04ae61..422c5a1 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -134,7 +134,7 @@ type(Array) -> len(Array) -> Array#array.len. --doc "Returns the length of an array.". +-doc "Returns the length of an element in an array.". -spec element_len(Array :: array()) -> Length :: pos_integer() | undefined. element_len(Array) -> Array#array.element_len. From 6a8cb004e03a7bb2a18fbe83bc1ce76c60f7fc66 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Mon, 20 Jul 2026 17:59:39 +0530 Subject: [PATCH 15/18] Fix variable size list layout link --- src/arrow_variable_list_array.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arrow_variable_list_array.erl b/src/arrow_variable_list_array.erl index 46cdc97..c8b6d8e 100644 --- a/src/arrow_variable_list_array.erl +++ b/src/arrow_variable_list_array.erl @@ -17,9 +17,9 @@ -module(arrow_variable_list_array). -moduledoc """ -Provides support for Arrow's Fixed-Size List Layout. +Provides support for Arrow's Variable-Size List Layout. -This module provides support for the [Fixed-Size List +This module provides support for the [Variable-Size List Layout](https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout), which is an layout that supports storing a list of lists of a specific length and nesting. From 36c85d5daef12d6122680c1802c50fd6d68d2d52 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Mon, 20 Jul 2026 18:06:11 +0530 Subject: [PATCH 16/18] Fix issues in arrow type --- src/arrow_type.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/arrow_type.erl b/src/arrow_type.erl index a239b01..1820461 100644 --- a/src/arrow_type.erl +++ b/src/arrow_type.erl @@ -62,11 +62,11 @@ Here you can find the name of each table: |------------------|-------|--------------| | Booleans | N/A | `bool` | | Signed Integer | `s` | `s64` | -| Unsigned Integer | `u ` | `u16 ` | -| Float | `f ` | `f32 ` | -| Binaries | N/A | `bin ` | +| Unsigned Integer | `u` | `u16` | +| Float | `f` | `f32` | +| Binaries | N/A | `bin` | -Types that are shorthand are under the `t:arrow_shorthand_type()` and +Types that are shorthand are under the `t:arrow_shorthand_type/0` and `arrow_short_*()` types. ### Longhand Syntax @@ -76,10 +76,10 @@ tuple is `{Name, Size}`. Thus, `Int 8` would be represented as `{s, 8}`. Do note that in the case or Booleans and Binaries their `Size` is `undefined`, as the have variable or undefined size. -Types that are longhand are under the `t:arrow_longhand_type()` and +Types that are longhand are under the `t:arrow_longhand_type/0` and `arrow_long_*()` types. -### Type +### Nested Types A Nested Type is any data structure that supports nesting. This can include Lists, FixedLists, Maps, Structs and more, but is curently limited to @@ -98,7 +98,7 @@ each element in an array. A list of `Int8` is 2 dimensional, and a list of ## Native Type A Native Type is any Erlang Datatype that can be serialized by `arrow`, -and is represented by the type `t:native_type()`. It currently only includes +and is represented by the type `t:native_type/0`. It currently only includes values that are primitive in nature such as booleans, integers, floats, binaries, and additionally the atoms `undefined` and `nil` to represent a `NULL` value. From a752fa9ba3c57ff04eb78cd504b4817a74a25ba8 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Mon, 20 Jul 2026 18:17:18 +0530 Subject: [PATCH 17/18] Add bitmap TODO --- src/arrow_bitmap.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/arrow_bitmap.erl b/src/arrow_bitmap.erl index 449a9df..28b743c 100644 --- a/src/arrow_bitmap.erl +++ b/src/arrow_bitmap.erl @@ -56,6 +56,9 @@ There are 5 important characteristics to remember about the validity bitmap: -doc """ Returns the Validity Bitmap along with the Null Count, of an Array. """. +%% TODO: Bitmap should be arrow_buffer:buffer() | undefined +%% Investigate why Dialyzer isn't raising, and potentially +%% run eqWAlizer via ELP on the CI. -spec validity_bitmap(Value :: [arrow_type:native_type()] | list()) -> {Bitmap :: arrow_buffer:buffer(), non_neg_integer()}. validity_bitmap(Value) -> From 2b074baec0ff85b62b8c68c3aeabc2dc12e2c858 Mon Sep 17 00:00:00 2001 From: Benjamin Philip Date: Mon, 20 Jul 2026 18:35:30 +0530 Subject: [PATCH 18/18] Fix spelling, grammar and formatting errors --- src/arrow_array.erl | 2 +- src/arrow_buffer.erl | 4 ++-- src/arrow_fixed_list_array.erl | 6 +++--- src/arrow_ipc_message.erl | 2 +- src/arrow_ipc_record_batch.erl | 8 +++++--- src/arrow_ipc_schema.erl | 7 ++++--- src/arrow_ipc_type.erl | 9 +++++---- src/arrow_offsets.erl | 4 ++-- src/arrow_type.erl | 8 ++++---- src/arrow_variable_list_array.erl | 2 +- 10 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/arrow_array.erl b/src/arrow_array.erl index 422c5a1..56a89a4 100644 --- a/src/arrow_array.erl +++ b/src/arrow_array.erl @@ -175,7 +175,7 @@ order: 2. `offsets` 3. `data` -In case an array doesn't have any of the following buffers, it is ommitted. +In case an array doesn't have any of the following buffers, it is omitted. (e.g. validity in arrays with a null count of 0, offsets in fixed primitive arrays). In the case of a nested array, `data` will be serialized form of nested array. diff --git a/src/arrow_buffer.erl b/src/arrow_buffer.erl index 0c6cff8..ecec6d5 100644 --- a/src/arrow_buffer.erl +++ b/src/arrow_buffer.erl @@ -22,7 +22,7 @@ -moduledoc """ Buffer implementation for `arrow`. -This module adds suppport for buffers, or Contiguous Memory Regions. There are +This module adds support for buffers, or Contiguous Memory Regions. There are multiple things to know about [buffers](https://arrow.apache.org/docs/format/Glossary.html#term-buffer): @@ -73,7 +73,7 @@ from_erlang(Data, Type) -> -doc """ Creates a new buffer from a list of Erlang values or binaries, given its -and length +type and length """. -spec from_erlang( Data :: [arrow_type:native_type()] | binary(), diff --git a/src/arrow_fixed_list_array.erl b/src/arrow_fixed_list_array.erl index bf2f238..7308d7c 100644 --- a/src/arrow_fixed_list_array.erl +++ b/src/arrow_fixed_list_array.erl @@ -21,14 +21,14 @@ Provides support for Arrow's Fixed-Size List Layout. This module provides support for the [Fixed-Size List Layout](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-list-layout), -which is an layout that supports storing a list of lists of a specific length -and nesting. +which is a layout that supports storing a list of lists of a specific length and +nesting. ## Invalid Input It is important that care is taken when passing input values to this module. For performance reasons, the input is not validated. The function crashes on nesting -that is inconsitent: a. with the type, b. between elements. The lists are +that is inconsistent: either with the type, or between elements. The lists are checked to have the same length as each other at the top level, but not for deeper levels. The behaviour of the module on invalid input **CANNOT BE GUARANTEED**. Therefore, one must be careful to not to **CRASH THE PROCESS** or diff --git a/src/arrow_ipc_message.erl b/src/arrow_ipc_message.erl index 18265e1..c27eb14 100644 --- a/src/arrow_ipc_message.erl +++ b/src/arrow_ipc_message.erl @@ -36,7 +36,7 @@ to represent a message. Metadata such as: or a binary (in the case of Record Batch). Currently, changing the version and custom metadata are not supported, but they -have been added for forwards comapatibility. +have been added for forwards compatibility. This module also provides the `to_ipc/1` function which serializes the message into the [Encapsulated Message diff --git a/src/arrow_ipc_record_batch.erl b/src/arrow_ipc_record_batch.erl index 0a3d517..6d1cd77 100644 --- a/src/arrow_ipc_record_batch.erl +++ b/src/arrow_ipc_record_batch.erl @@ -17,9 +17,11 @@ -module(arrow_ipc_record_batch). -moduledoc """ -Provides a record and functions to deal with RecordBatches +Provides a record and functions to deal with RecordBatches. + + A [RecordBatch](https://github.com/apache/arrow/blob/16328f0ccc73b7df665b4a18feb6adf26b7aa0e2/format/Message.fbs#L81-L102) represents a list of equal length arrays and their -coresponding buffers. This module provides a record and a function to manage +corresponding buffers. This module provides a record and a function to manage all the metadata required to represent a RecordBatch. Metadata such as: 1. `length`: The number of rows or records. In other words, the @@ -31,7 +33,7 @@ all the metadata required to represent a RecordBatch. Metadata such as: array. 4. `compression`: The compression applied on the body of the Record - Batch. Can either by `undefined` (i.e. no compression), `zstd` for + Batch. Can either be `undefined` (i.e. no compression), `zstd` for [Zstandard](https://facebook.github.io/zstd/), or `lz4_frame` for [LZ4 Frame](https://android.googlesource.com/platform/external/lz4/+/HEAD/doc/lz4_Frame_format.md). Defaults to `undefined`. diff --git a/src/arrow_ipc_schema.erl b/src/arrow_ipc_schema.erl index 3e56f39..072a02a 100644 --- a/src/arrow_ipc_schema.erl +++ b/src/arrow_ipc_schema.erl @@ -17,8 +17,9 @@ -module(arrow_ipc_schema). -moduledoc """ -Provides a record and functions to deal with Schemas A -[Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530)[1] +Provides a record and functions to deal with Schemas. + +A [Schema](https://github.com/apache/arrow/blob/3456131ab7350bee5d9569ffd63d3f0ee713991c/format/Schema.fbs#L514-L530) represents a table, or a list of arrays of equal length. This module provides a record and a function to manage all the metadata required to represent a schema. Metadata such as: @@ -32,7 +33,7 @@ Metadata such as: present in other implementations of Arrow. Currently, big endianness, custom metadata and features are not supported, but -they have been added for forwards comapatibility. +they have been added for forwards compatibility. You can find Schemas in the Arrow spec [here](https://arrow.apache.org/docs/format/Columnar.html#schema-message). diff --git a/src/arrow_ipc_type.erl b/src/arrow_ipc_type.erl index 5e4e172..def4957 100644 --- a/src/arrow_ipc_type.erl +++ b/src/arrow_ipc_type.erl @@ -17,14 +17,15 @@ -module(arrow_ipc_type). -moduledoc """ -Provides types and functions to work with IPC types. This module provides -functions and types to produce the types in IPC Schema +Provides types and functions to work with IPC types. + +This module provides functions and types to produce the types in IPC Schema [definitions](https://github.com/apache/arrow/blob/main/format/Schema.fbs). These types are generated according to `Type` union in the schema. The types have been represented in the form `{TypeName, Metadata}`, where `TypeName` is the name of the type and is an atom, and `Metadata` is a map of all the metadata associated with it. In case a type has no metadata associated with it, it is -represented as just `TypeName` +represented as just `TypeName`. """. -export([from_erlang/1]). -export_type([ @@ -88,7 +89,7 @@ represented as just `TypeName` %%%%%%%%%%%%%%%%%%% -doc """ -Returns the IPC Type for an `t:arrow_array:array()`. +Returns the IPC Type for an `t:arrow_array:array/0`. """. -spec from_erlang(Array :: arrow_array:array()) -> Type :: ipc_type(). from_erlang(Array) -> diff --git a/src/arrow_offsets.erl b/src/arrow_offsets.erl index 4e1d49b..9fef0b0 100644 --- a/src/arrow_offsets.erl +++ b/src/arrow_offsets.erl @@ -28,8 +28,8 @@ support for generating offsets. There are couple of things to remember about offsets: -Firstly, each element in the offsets coresponds to the distance in bytes of the -coresponding element in the values from the beginning of the buffer. +Firstly, each element in the offsets corresponds to the distance in bytes of the +corresponding element in the values from the beginning of the buffer. I.E., distance of `values[j]` from beginning of the buffer = `offsets[j]`. diff --git a/src/arrow_type.erl b/src/arrow_type.erl index 1820461..899e96c 100644 --- a/src/arrow_type.erl +++ b/src/arrow_type.erl @@ -20,10 +20,10 @@ Provides representation for as well as functions for working with Arrow specific datatypes. -This module provides a longhand (a 2 tuple) as well as a shorthand (an atom) -represenations for representing a primitive types. Additionally it represents -Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types -that are supported by this package). +This module provides longhand (a 2 tuple) as well as shorthand (an atom) +representations for representing primitive types. Additionally it represents +Nested Types (i.e. Lists) as well as Native Types (i.e. Erlang Native Types that +are supported by this package). Generally, when we say an array A has type X, we mean that each element in A has type X. Thus, an array of type of `Int8` would look like `[1, 2, 3]`, but diff --git a/src/arrow_variable_list_array.erl b/src/arrow_variable_list_array.erl index c8b6d8e..630954d 100644 --- a/src/arrow_variable_list_array.erl +++ b/src/arrow_variable_list_array.erl @@ -28,7 +28,7 @@ and nesting. It is important that care is taken when passing input values to this module. For performance reasons, the input is not validated. The -function crashes on nesting that is inconsitent: a. with the type, b. +function crashes on nesting that is inconsistent: a. with the type, b. between elements. The behaviour on invalid input **CANNOT BE GUARANTEED**. Therefore, one must be careful to not to **CRASH THE PROCESS** or worse still, **PRODUCE INVALID OUTPUT**.