Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,13 @@ endif ()

add_custom_target(build_all_plugins)

# Declared here so add_compile_definitions reaches all sibling nes-* targets.
# nes-plugins/CMakeLists.txt re-declares the same option (no-op when cached).
option(NES_ENABLE_MEOS "Enable MEOS plugin (requires libmeos installed on the system)" ON)
if(NES_ENABLE_MEOS)
add_compile_definitions(NES_ENABLE_MEOS)
endif()

# Add target for common lib, which contains a minimal set
# of shared functionality used by all components of nes
file(GLOB NES_DIRECTORIES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "nes-*")
Expand Down
4 changes: 4 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@
tbb
python3
openjdk21
paho-mqtt-c
paho-mqtt-cpp
]) ++ [ follyPkg antlr4Pkg ];

antlr4Jar = pkgs.fetchurl {
Expand Down Expand Up @@ -244,6 +246,7 @@
"-DNES_ENABLES_TESTS=ON"
"-DCMAKE_MODULE_PATH=${libdwarfModule}/share/cmake/Modules"
"-DANTLR4_JAR_LOCATION=${antlr4Jar}"
"-DNES_ENABLE_MEOS=OFF"
];

enableParallelBuilding = true;
Expand Down Expand Up @@ -347,6 +350,7 @@
"-DLLVM_DIR=${commonCmakeEnv.LLVM_DIR}"
"-DANTLR4_JAR_LOCATION=${antlr4Jar}"
"-DCMAKE_MODULE_PATH=${libdwarfModule}/share/cmake/Modules"
"-DNES_ENABLE_MEOS=OFF"
];
shellHook = ''
unset NES_PREBUILT_VCPKG_ROOT
Expand Down
6 changes: 5 additions & 1 deletion nes-physical-operators/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@ get_source(nes-physical-operators NES_PHYSICAL_OPERATORS_SOURCE_FILES)

# Add Library
add_library(nes-physical-operators ${NES_PHYSICAL_OPERATORS_SOURCE_FILES})
target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus nes-meos)
if(NES_ENABLE_MEOS)
target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus nes-meos)
else()
target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus)
endif()
target_include_directories(nes-physical-operators PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:include/nebulastream/>)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

if(NES_ENABLE_MEOS)
add_plugin(TemporalSequence AggregationPhysicalFunction nes-physical-operators TemporalSequenceAggregationPhysicalFunction.cpp)
add_plugin(Var AggregationPhysicalFunction nes-physical-operators VarAggregationFunction.cpp)
endif()
2 changes: 2 additions & 0 deletions nes-physical-operators/src/Functions/Meos/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.

if(NES_ENABLE_MEOS)
add_plugin(TemporalIntersects PhysicalFunction nes-physical-operators TemporalIntersectsPhysicalFunction.cpp)
add_plugin(TemporalIntersectsGeometry PhysicalFunction nes-physical-operators TemporalIntersectsGeometryPhysicalFunction.cpp)
add_plugin(TemporalAIntersectsGeometry PhysicalFunction nes-physical-operators TemporalAIntersectsGeometryPhysicalFunction.cpp)
add_plugin(TemporalEContainsGeometry PhysicalFunction nes-physical-operators TemporalEContainsGeometryPhysicalFunction.cpp)
add_plugin(TemporalEDWithinGeometry PhysicalFunction nes-physical-operators TemporalEDWithinGeometryPhysicalFunction.cpp)
add_plugin(TemporalAtStBox PhysicalFunction nes-physical-operators TemporalAtStBoxPhysicalFunction.cpp)
endif()
55 changes: 29 additions & 26 deletions nes-plugins/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,33 +20,36 @@ activate_optional_plugin("Sources/TCPSource" ON)
# Enable the Generator source plugin; required by systests and repl tests
activate_optional_plugin("Sources/GeneratorSource" ON)
activate_optional_plugin("Sinks/VoidSink" ON)
activate_optional_plugin("Sources/MQTTSource" ON)
activate_optional_plugin("Sinks/MQTTSink" ON)

# MEOS is a dependency
activate_optional_plugin("MEOS" ON)
message(STATUS "Enable MEOS Plugin")

# Detect the platform
if (APPLE)
message(STATUS "Building on macOS")
# Set the include and library directories for Homebrew (macOS)
set(MEOS_INCLUDE_DIR "/opt/homebrew/include" CACHE PATH "Path to MEOS include directory")
set(MEOS_PLUGIN_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include" CACHE PATH "Path to MEOS plugin include directory")
include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
link_directories(${MEOS_LIBRARY_DIR} )

else()
message(STATUS "Building on Linux")

# Default behavior for Linux
find_package(meos REQUIRED)
if(meos_FOUND)
message(STATUS "MEOS found: include=${meos_INCLUDE_DIR}, library=${meos}")
include_directories(SYSTEM ${meos_INCLUDE_DIR})
option(NES_ENABLE_MQTT "Enable MQTT source and sink plugins (requires PahoMqttCpp)" ON)
activate_optional_plugin("Sources/MQTTSource" ${NES_ENABLE_MQTT})
activate_optional_plugin("Sinks/MQTTSink" ${NES_ENABLE_MQTT})

option(NES_ENABLE_MEOS "Enable MEOS plugin (requires libmeos installed on the system)" ON)
activate_optional_plugin("MEOS" ${NES_ENABLE_MEOS})
if (NES_ENABLE_MEOS)
message(STATUS "Enable MEOS Plugin")

# Detect the platform
if (APPLE)
message(STATUS "Building on macOS")
# Set the include and library directories for Homebrew (macOS)
set(MEOS_INCLUDE_DIR "/opt/homebrew/include" CACHE PATH "Path to MEOS include directory")
set(MEOS_PLUGIN_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include" CACHE PATH "Path to MEOS plugin include directory")
include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
link_directories(${MEOS_LIBRARY_DIR} )

else()
message(FATAL_ERROR "MEOS library not found")
message(STATUS "Building on Linux")

# Default behavior for Linux
find_package(meos REQUIRED)
if(meos_FOUND)
message(STATUS "MEOS found: include=${meos_INCLUDE_DIR}, library=${meos}")
include_directories(SYSTEM ${meos_INCLUDE_DIR})
else()
message(FATAL_ERROR "MEOS library not found")
endif()
endif()
endif()

Expand Down
9 changes: 7 additions & 2 deletions nes-plugins/Sinks/MQTTSink/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,10 @@ target_include_directories(mqtt_sink_validation_plugin_library
)

find_package(PahoMqttCpp CONFIG REQUIRED)
target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
if(TARGET PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
else()
target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
endif()
9 changes: 7 additions & 2 deletions nes-plugins/Sources/MQTTSource/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,10 @@ target_include_directories(mqtt_source_validation_plugin_library
)

find_package(PahoMqttCpp CONFIG REQUIRED)
target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
if(TARGET PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
else()
target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
endif()
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,10 @@
#include <QueryExecutionConfiguration.hpp>
#include <RewriteRuleRegistry.hpp>
// Special-case lowering for TEMPORAL_SEQUENCE (multi-input) aggregation
#ifdef NES_ENABLE_MEOS
#include <Operators/Windows/Aggregations/Meos/TemporalSequenceAggregationLogicalFunctionV2.hpp>
#include <Aggregation/Function/Meos/TemporalSequenceAggregationPhysicalFunction.hpp>
#endif

namespace NES
{
Expand Down Expand Up @@ -128,6 +130,7 @@ getAggregationPhysicalFunctions(const WindowedAggregationLogicalOperator& logica
const auto name = descriptor->getName();

// Custom lowering path for TEMPORAL_SEQUENCE: needs three field functions (lon, lat, ts)
#ifdef NES_ENABLE_MEOS
if (name == std::string_view("TemporalSequence"))
{
auto tsDescriptor = std::dynamic_pointer_cast<TemporalSequenceAggregationLogicalFunctionV2>(descriptor);
Expand Down Expand Up @@ -159,6 +162,7 @@ getAggregationPhysicalFunctions(const WindowedAggregationLogicalOperator& logica
aggregationPhysicalFunctions.push_back(std::move(phys));
continue;
}
#endif

// Default path: use registry for single-input aggregations
auto aggregationInputFunction = QueryCompilation::FunctionProvider::lowerFunction(descriptor->onField);
Expand Down
2 changes: 2 additions & 0 deletions tools/codegen/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
__pycache__/
*.pyc
87 changes: 87 additions & 0 deletions tools/codegen/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
# MobilityNebula MEOS per-event operator codegen

This directory contains the IDL-driven code generator for the canonical
615-op per-event MEOS surface in MobilityNebula.

## What it is

The generator reads JSON descriptor files that describe each MEOS operator
(name, SQL token, MEOS C function, argument types, return type) and emits
the four C++ files that NES requires per operator:

- `nes-logical-operators/include/Functions/Meos/<Name>LogicalFunction.hpp`
- `nes-logical-operators/src/Functions/Meos/<Name>LogicalFunction.cpp`
- `nes-physical-operators/include/Functions/Meos/<Name>PhysicalFunction.hpp`
- `nes-physical-operators/src/Functions/Meos/<Name>PhysicalFunction.cpp`

In addition to the per-operator files, the generator injects the
CMakeLists.txt target lists, the AntlrSQL.g4 token rules, and the
AntlrSQLQueryPlanCreator.cpp dispatch table so the full NES build stays
consistent with the committed surface.

## Pipeline

```
MobilityDB master
└─► MEOS-API run.py <headers> # libclang extracts the C API
└─► meos-idl-master.json # IDL: all public MEOS functions
└─► build_descriptor.py # author a JSON descriptor family
└─► *-descriptor.json # per-family operator list
└─► gen_all.py # orchestrates all 9 families
└─► 4 C++ files per op + CMake/grammar/QPC glue
```

`meos-idl-master.json` committed here is a snapshot of the IDL generated
from the current MobilityDB master. Regenerate it via MEOS-API when the
upstream C surface changes.

## Descriptor families (9 total, 615 operators)

| Descriptor file | Family |
|-----------------------------------------|---------------------------------------------|
| `tfloat-transforms-descriptor.json` | tfloat trig / abs / shift-scale |
| `numeric-descriptor.json` | tint/tfloat/tbigint arithmetic, comparisons |
| `spatial-predicates-descriptor.json` | tgeo/tcbuffer/tnpoint/tpose spatial preds |
| `spatial-phasec-full-descriptor.json` | full phase-C tgeo/tcbuffer/trgeometry preds |
| `phasec-rest-descriptor.json` | remaining phase-C spatial ops |
| `trgeo-descriptor.json` | trgeometry spatial predicates |
| `phasec-composed-descriptor.json` | composed / multi-step spatial ops |
| `missing-ops-descriptor.json` | 69 canonical ops not covered above |
| `canonical-new-ops-descriptor.json` | 85 gap-fill ops closing the TARGET_SURFACE |

Intentionally excluded: bbox/topological operators (stbox/stbox,
stbox/tspatial, tbox/tnumber families) and non-canonical per-type numeric
two-temporal comparisons are out of scope for the 615-op canonical surface.

## How to regenerate after a MobilityDB API change

```sh
python tools/codegen/gen_all.py --output-root <repo-root>
```

`<repo-root>` must be the MobilityNebula repository root (the directory
that contains `nes-logical-operators/`). The script writes all 2 460 C++
files and updates CMakeLists.txt, AntlrSQL.g4, and
AntlrSQLQueryPlanCreator.cpp in place. Commit the result.

## Files in this directory

| File | Purpose |
|-------------------------------|---------------------------------------------------|
| `codegen_nebula.py` | Core emitter: templates for all 4 C++ file kinds |
| `gen_all.py` | Orchestrator: runs all 9 descriptor families |
| `codegen_aggregations.py` | Supplementary generator for aggregation operators |
| `build_descriptor.py` | Helper: build a new descriptor from IDL entries |
| `codegen_input.example.json` | Annotated example descriptor for new families |
| `meos-idl-master.json` | Committed IDL snapshot from MobilityDB master |
| `build_local.sh` | Local convenience wrapper for gen_all.py |
| `*-descriptor.json` | The 9 operator family descriptors |

## Drift guard

`.github/workflows/codegen-drift-guard.yml` (added in the follow-up PR)
runs on every pull request. It executes `gen_all.py --output-root .` and
then asserts `git diff --quiet`. Any divergence between the committed
operator files and the generator output fails the check and prints the
diff stat. To fix a drift failure, run the regeneration command above and
commit the updated files.
Loading
Loading