Generate the asHexWKB output surface for temporal types

.github/workflows/generate.yml

@@ -0,0 +1,72 @@
+#
+# Derives the MEOS catalog from MobilityDB in CI (instead of committing a
+# meos-idl.json snapshot) and verifies that the committed generated UDF surface
+# still matches a fresh derivation. This is the shared ecosystem model: bindings
+# stop vendoring the catalog and derive it from one MobilityDB commit via the
+# MEOS-API provision-meos action (the JMEOS/Spark/Flink/Kafka pattern).
+#
+name: Derive catalog and verify generated code
+
+on:
+  pull_request:
+    branches:
+      - main
+      - 'feat/**'
+      - 'fix/**'
+    paths-ignore:
+      - '**/README.md'
+      - 'doc/**'
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**/README.md'
+      - 'doc/**'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}
+  cancel-in-progress: true
+
+jobs:
+  derive-and-verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # The catalog is derived from the SAME MobilityDB commit the vcpkg MEOS
+      # port builds libmeos from, so the generated code always matches the
+      # libmeos it is compiled against. tools/meos-source-commit.txt is the
+      # single source of truth for that commit (the vcpkg portfile reads the same
+      # file), mirroring the shared ecosystem provision-meos model.
+      - name: Resolve the MobilityDB source commit
+        id: ref
+        run: echo "sha=$(tr -d '[:space:]' < tools/meos-source-commit.txt)" >> "$GITHUB_OUTPUT"
+
+      # Generate meos-idl.json from that commit via the shared MEOS-API action.
+      # Catalog only (build-libmeos: false): the generator is pure Python and
+      # reads the catalog JSON, so no libmeos build is needed here.
+      - name: Provision the MEOS catalog
+        id: provision
+        uses: MobilityDB/MEOS-API/.github/actions/provision-meos@master
+        with:
+          mobilitydb-ref: ${{ steps.ref.outputs.sha }}
+          build-libmeos: "false"
+
+      - name: Stage the catalog
+        run: |
+          mkdir -p tools/catalog
+          cp "${{ steps.provision.outputs.catalog-path }}" tools/catalog/meos-idl.json
+
+      # Regenerate the committed UDF surface and fail if it drifts. The committed
+      # src/generated/*.cpp (compiled by the distribution pipeline) must always
+      # equal a fresh derivation from the current catalog.
+      - name: Regenerate and check for drift
+        run: |
+          python3 tools/codegen_duck_udfs.py \
+            tools/catalog/meos-idl.json \
+            src/generated/generated_temporal_udfs.cpp
+          if ! git diff --exit-code src/generated/; then
+            echo "::error::src/generated is stale — re-run tools/codegen_duck_udfs.py against a freshly derived catalog and commit the result"
+            exit 1
+          fi

.gitignore

@@ -5,7 +5,11 @@ duckdb_unittest_tempdir/
 .DS_Store
 testext
 test/python/__pycache__/
+tools/__pycache__/
 .Rhistory
 vcpkg/
 *.log
 examples/quickstart/*.parquet
+
+# Derived in CI from MobilityDB via MEOS-API provision-meos (see .github/workflows/generate.yml); not vendored.
+tools/catalog/meos-idl.json

CMakeLists.txt

@@ -37,6 +37,23 @@ endif()
 # MEOS from overlay port
 find_package(MEOS CONFIG REQUIRED)
 
+# H3 headers: the all-families libmeos exposes meos_h3.h, which includes
+# <h3api.h>. SIBLING-FROM MobilityDB CMakeLists.txt H3 block (find_path
+# H3_INCLUDE_DIR NAMES h3api.h PATH_SUFFIXES h3) so code compiling against
+# meos_h3.h sees the system libh3 headers, just as the meos port (-DH3=ON) does.
+# The dir is attached to the extension targets below (mirroring the json-c
+# block) because build_*_extension targets don't inherit directory includes.
+# SIBLING-FROM MobilityDB CMakeLists.txt H3 block; NO_CMAKE_FIND_ROOT_PATH + explicit
+# system PATHS are the binding-side adaptation: under the vcpkg toolchain find_path is
+# confined to the vcpkg root and never sees the system /usr/include/h3 that libh3-dev ships.
+find_path(H3_INCLUDE_DIR NAMES h3api.h
+  PATH_SUFFIXES h3
+  PATHS /usr/include /usr/local/include
+  NO_CMAKE_FIND_ROOT_PATH)
+if(NOT H3_INCLUDE_DIR)
+  message(FATAL_ERROR "h3api.h not found; install libh3-dev (the meos port is built with -DH3=ON).")
+endif()
+
 if(TARGET GEOS::geos_c)
   set(GEOS_TGT GEOS::geos_c)
 elseif(TARGET GEOS::geos)
@@ -57,7 +74,9 @@ include_directories(duckdb-spatial/src/spatial)
 
 set(EXTENSION_SOURCES
     src/spatial_ref_sys_csv.inc
-    src/mobilityduck_extension.cpp        
+    src/mobilityduck_extension.cpp
+    src/generated/generated_temporal_udfs.cpp
+    src/generated/generated_type_registration.cpp
     src/temporal/temporal.cpp
     src/temporal/temporal_functions.cpp
     src/temporal/temporal_aggregates.cpp
@@ -89,6 +108,8 @@ set(EXTENSION_SOURCES
     src/geo/tgeogpoint.cpp
     src/geo/tgeogpoint_in_out.cpp
     src/geo/tgeogpoint_ops.cpp
+    src/quadbin/tquadbin.cpp
+    src/cbuffer/tcbuffer.cpp
     src/index/rtree_module.cpp
     src/single_tile_getters.cpp
     src/index/rtree_index_create_physical.cpp
@@ -111,6 +132,11 @@ else()
   set(JSONC_TGT ${JSON-C_LIBRARIES})
 endif()
 
+# H3 headers for meos_h3.h (see the find_path above) — attached per-target like
+# json-c, since build_*_extension targets don't inherit directory includes.
+target_include_directories(${EXTENSION_NAME} PRIVATE ${H3_INCLUDE_DIR})
+target_include_directories(${LOADABLE_EXTENSION_NAME} PRIVATE ${H3_INCLUDE_DIR})
+
 # -----------------------------
 # Link libraries 
 # -----------------------------

Makefile

@@ -12,29 +12,39 @@ include extension-ci-tools/makefiles/duckdb_extension.Makefile
 # "TimeZone") to Europe/Brussels.  Tests pass on any OS timezone — the
 # extension is the single source of truth, no TZ env var needed.
 #
-# LoadInternal also calls ExtensionHelper::AutoLoadExtension(db, "icu") so
-# the timezone option is honoured. Autoload looks for the extension on disk
-# at $HOME/.duckdb/extensions/<duckdb_version>/<platform>/icu.duckdb_extension
-# and falls back to a hub download. That fails both inside the linux_amd64
-# test docker container (empty path, no network egress) and on the macOS
-# osx_arm64 test runner (hub icu not reliably resolvable). We copy the
-# icu.duckdb_extension that was built locally as part of this extension's
-# build (declared in extension_config.cmake) into the expected path,
-# matched to the DuckDB platform string, before running the unittester.
+# LoadInternal also auto-loads ICU so the named "Europe/Brussels" zone
+# resolves. DuckDB autoloads ICU from
+#   $HOME/.duckdb/extensions/<version>/<platform>/icu.duckdb_extension
+# and otherwise falls back to a hub download — which fails inside the CI test
+# docker (empty path, no network egress) and on the macOS osx_arm64 runner
+# (hub ICU not reliably resolvable). So we stage the locally-built ICU into the
+# expected path before the unittester runs.
+#
+# Target DuckDB is the v1.4.x LTS line, with later versions (v1.5.x) supported
+# in a multi-version matrix the same way MobilityDB supports PostgreSQL 13-18 —
+# so the staging path must NOT hardcode the version or platform. We derive both
+# from the freshly-built duckdb binary (authoritative for whatever is being
+# tested); DUCKDB_VERSION_TAG and the uname map are fallbacks only.
 DUCKDB_VERSION_TAG := v1.4.4
 
 define stage_icu
 	@if [ -f ./build/$(1)/extension/icu/icu.duckdb_extension ]; then \
-	  case "$$(uname -s)-$$(uname -m)" in \
-	    Linux-x86_64)  platform=linux_amd64 ;; \
-	    Linux-aarch64) platform=linux_arm64 ;; \
-	    Darwin-arm64)  platform=osx_arm64 ;; \
-	    Darwin-x86_64) platform=osx_amd64 ;; \
-	    *)             platform=$$(uname -m) ;; \
-	  esac; \
-	  target=$$HOME/.duckdb/extensions/$(DUCKDB_VERSION_TAG)/$$platform; \
+	  duckdb_bin=./build/$(1)/duckdb; \
+	  version_tag=$$( [ -x "$$duckdb_bin" ] && "$$duckdb_bin" --version 2>/dev/null | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1 ); \
+	  platform=$$( [ -x "$$duckdb_bin" ] && echo 'PRAGMA platform;' | "$$duckdb_bin" -noheader -list 2>/dev/null | tr -d '[:space:]' ); \
+	  [ -n "$$version_tag" ] || version_tag=$(DUCKDB_VERSION_TAG); \
+	  if [ -z "$$platform" ]; then \
+	    case "$$(uname -s)-$$(uname -m)" in \
+	      Linux-x86_64)  platform=linux_amd64 ;; \
+	      Linux-aarch64) platform=linux_arm64 ;; \
+	      Darwin-arm64)  platform=osx_arm64 ;; \
+	      Darwin-x86_64) platform=osx_amd64 ;; \
+	      *)             platform=$$(uname -m) ;; \
+	    esac; \
+	  fi; \
+	  target=$$HOME/.duckdb/extensions/$$version_tag/$$platform; \
 	  mkdir -p "$$target" && cp -f ./build/$(1)/extension/icu/icu.duckdb_extension "$$target/" && \
-	  echo "Staged icu.duckdb_extension at $$target/"; \
+	  echo "Staged icu.duckdb_extension at $$target/ (duckdb $$version_tag / $$platform)"; \
 	fi
 endef
 

src/cbuffer/tcbuffer.cpp

@@ -0,0 +1,169 @@
+/* MobilityDuck binding for the MEOS circular-buffer types (cbuffer + tcbuffer).
+ *
+ * cbuffer is a varlena value (a point + a radius), surfaced as a BLOB.
+ * tcbuffer is the temporal circular buffer, stored as a Temporal* blob (BLOB).
+ *
+ * Only the type registration + text I/O boundary is hand-written here; the
+ * operation surface is generated from the MEOS-API catalog into src/generated/.
+ */
+
+#include "cbuffer/tcbuffer.hpp"
+#include "temporal/temporal.hpp"
+#include "tydef.hpp"
+#include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/main/extension/extension_loader.hpp"
+#include "duckdb/common/extension_type_info.hpp"
+#include "duckdb/function/scalar_function.hpp"
+#include "mobilityduck/meos_exec_serial.hpp"
+#include "time_util.hpp"
+
+extern "C" {
+    #include <meos.h>
+    #include <meos_cbuffer.h>
+    #include <meos_internal.h>
+}
+
+#include <cstdlib>
+#include <cstring>
+#include <string>
+
+namespace duckdb {
+
+LogicalType CbufferTypes::cbuffer() {
+    auto type = LogicalType(LogicalTypeId::BLOB);
+    type.SetAlias("cbuffer");
+    return type;
+}
+
+LogicalType CbufferTypes::tcbuffer() {
+    auto type = LogicalType(LogicalTypeId::BLOB);
+    type.SetAlias("tcbuffer");
+    return type;
+}
+
+void CbufferTypes::RegisterTypes(ExtensionLoader &loader) {
+    loader.RegisterType("cbuffer",  cbuffer());
+    loader.RegisterType("tcbuffer", tcbuffer());
+}
+
+namespace {
+
+/* cbuffer is a varlena value: the stored BLOB holds the full varlena bytes, so
+ * copy-in by the blob size and copy-out by VARSIZE_ANY. */
+inline Cbuffer *BlobToCbuffer(string_t blob) {
+    size_t sz = blob.GetSize();
+    void *copy = malloc(sz);
+    memcpy(copy, blob.GetData(), sz);
+    return reinterpret_cast<Cbuffer *>(copy);
+}
+
+inline string_t CbufferToBlob(Vector &result, Cbuffer *cb) {
+    if (!cb) return string_t();
+    /* Cbuffer is a 4-byte-header varlena (int32 vl_len_); VARSIZE reads it. */
+    size_t sz = VARSIZE(cb);
+    string_t out = StringVector::AddStringOrBlob(
+        result, string_t(reinterpret_cast<const char *>(cb), sz));
+    free(cb);
+    return out;
+}
+
+inline Temporal *BlobToTemp(string_t blob) {
+    size_t sz = blob.GetSize();
+    uint8_t *copy = (uint8_t *) malloc(sz);
+    memcpy(copy, blob.GetData(), sz);
+    return reinterpret_cast<Temporal *>(copy);
+}
+
+inline string_t TempToBlob(Vector &result, Temporal *t) {
+    if (!t) return string_t();
+    size_t sz = temporal_mem_size(t);
+    string_t out = StringVector::AddStringOrBlob(
+        result, string_t(reinterpret_cast<const char *>(t), sz));
+    free(t);
+    return out;
+}
+
+} // namespace
+
+/* =====================================================================
+ * In / out — static cbuffer value
+ * ===================================================================== */
+
+bool CbufferFunctions::Cbuffer_in_cast(
+    Vector &source, Vector &result, idx_t count, CastParameters &parameters)
+{
+    UnaryExecutor::Execute<string_t, string_t>(
+        source, result, count,
+        [&](string_t s) -> string_t {
+            std::string str(s.GetData(), s.GetSize());
+            Cbuffer *cb = cbuffer_in(str.c_str());
+            return CbufferToBlob(result, cb);
+        });
+    return true;
+}
+
+bool CbufferFunctions::Cbuffer_out_cast(
+    Vector &source, Vector &result, idx_t count, CastParameters &parameters)
+{
+    UnaryExecutor::Execute<string_t, string_t>(
+        source, result, count,
+        [&](string_t blob) -> string_t {
+            Cbuffer *cb = BlobToCbuffer(blob);
+            char *str = cbuffer_out(cb, OUT_DEFAULT_DECIMAL_DIGITS);
+            free(cb);
+            std::string copy(str);
+            free(str);
+            return StringVector::AddString(result, copy);
+        });
+    return true;
+}
+
+/* =====================================================================
+ * In / out — tcbuffer temporal value
+ * ===================================================================== */
+
+bool CbufferFunctions::Tcbuffer_in_cast(
+    Vector &source, Vector &result, idx_t count, CastParameters &parameters)
+{
+    UnaryExecutor::Execute<string_t, string_t>(
+        source, result, count,
+        [&](string_t s) -> string_t {
+            std::string str(s.GetData(), s.GetSize());
+            Temporal *t = tcbuffer_in(str.c_str());
+            return TempToBlob(result, t);
+        });
+    return true;
+}
+
+bool CbufferFunctions::Tcbuffer_out_cast(
+    Vector &source, Vector &result, idx_t count, CastParameters &parameters)
+{
+    UnaryExecutor::Execute<string_t, string_t>(
+        source, result, count,
+        [&](string_t blob) -> string_t {
+            Temporal *t = BlobToTemp(blob);
+            char *str = temporal_out(t, OUT_DEFAULT_DECIMAL_DIGITS);
+            free(t);
+            std::string copy(str);
+            free(str);
+            return StringVector::AddString(result, copy);
+        });
+    return true;
+}
+
+/* =====================================================================
+ * Registration
+ * ===================================================================== */
+
+void CbufferTypes::RegisterCastFunctions(ExtensionLoader &loader) {
+    RegisterMeosCastFunction(loader, LogicalType::VARCHAR, cbuffer(),
+        CbufferFunctions::Cbuffer_in_cast);
+    RegisterMeosCastFunction(loader, cbuffer(), LogicalType::VARCHAR,
+        CbufferFunctions::Cbuffer_out_cast);
+    RegisterMeosCastFunction(loader, LogicalType::VARCHAR, tcbuffer(),
+        CbufferFunctions::Tcbuffer_in_cast);
+    RegisterMeosCastFunction(loader, tcbuffer(), LogicalType::VARCHAR,
+        CbufferFunctions::Tcbuffer_out_cast);
+}
+
+} // namespace duckdb