diff --git a/CHANGELOG.md b/CHANGELOG.md index be9aba67..e9f3c1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,52 @@ # Changelog +## 0.9.0-dev.10 — llama.cpp b10182, load-mode API, isolate log-callback fix + +Native rebuild required — `src/llama.cpp` moved `d6d0ce82` → `afeebe10` +(tag `b10182`), about seven weeks of upstream work. + +### Changed + +- Adapted to upstream `e6dd0e29a`, which collapsed the `use_mmap` / + `use_direct_io` / `use_mlock` booleans in `llama_model_params` into a + single `llama_load_mode` enum. **`ModelParams` keeps its three + booleans** — they are mapped at the FFI boundary, so callers are + unaffected. One semantic caveat: the enum has no direct-I/O-plus-mlock + value, so when both are requested direct I/O wins and mlock is dropped. + `useDirectIo` keeps its documented precedence over `useMmap`. +- Regenerated FFI bindings against the new pin. New upstream C API now + reachable but not yet wrapped: `llama_model_n_layer_nextn`, + `llama_model_ftype`, `llama_ftype_name`, + `llama_vocab_get_suppress_tokens`, and the mtmd batch-encoding API + (`mtmd_batch_init` / `_add_chunk` / `_encode` / `_get_output_embd`). +- `mtmd_encode` is deprecated upstream in favor of `mtmd_encode_chunk`. + This package reaches multimodal via `mtmd_helper_eval_chunks` and never + called it, so no change was needed. + +### Fixed + +- **`LlamaLibrary.dispose` now clears the log callback.** + `LlamaLog.silence` installs a `Pointer.fromFunction` bound to the + isolate that registered it, but the slot it occupies lives in + process-global llama.cpp/ggml state and outlives that isolate. The + stale pointer stayed installed, so the next isolate to emit a log line + invoked a callback owned by a dead isolate and the VM aborted with + "Cannot invoke native callback from a different isolate". Surfaced by + Dart 3.12's stricter cross-isolate check. + + Known remaining issue: parallel `dart test` still hits the concurrent + variant of this race, where one isolate holds a live callback while + another loads a model. Run the model-backed suite with `-j 1` until + `silence()` stops using a Dart callback altogether. + +### Tooling + +- `ffigen` 20.1.1 → 21.0.0, `lints` 5.0.0 → 6.1.0 (dev dependencies). + Note ffigen 21 requires Dart SDK ≥ 3.10 to run the generator; the + package's own `sdk: ^3.5.0` constraint for consumers is unchanged. +- Fixed the ffigen `-resource-dir` compiler-opt, which pointed at a clang + 17 toolchain directory that no longer exists. + ## 0.9.0-dev.9 — Gemma-4, MTP removed, dynamic Apple framework Native rebuild required — `src/llama.cpp` moved `6b4e4bd58` → `d6d0ce82` diff --git a/README.md b/README.md index 7f4604e3..ffb12ae9 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Then download the platform binary for your project: | Platform | Artifact | Where to put it | |---|---|---| | macOS (dev/test) | `libllama.dylib` + sibling `libggml*.dylib`, `libmtmd.dylib` | anywhere on disk; pass path to `LlamaEngine.spawn` | -| iOS / macOS app | `llama.xcframework` (3 slices: `ios-arm64`, `ios-arm64-simulator`, `macos-arm64`) | drag into Xcode → "Embed & Sign" → call `LlamaEngine.spawnFromProcess` | +| iOS / macOS app | `Llama.xcframework` (3 self-contained slices: `ios-arm64`, `ios-arm64_x86_64-simulator` [universal fat binary], `macos-arm64`) | Auto-linked via Flutter CocoaPods or drag into Xcode → "Embed & Sign" → call `LlamaEngine.spawnFromProcess` | | Android | `llama-cpp-dart.aar` (CPU + mtmd, arm64-v8a) **or** `llama-cpp-dart-hexagon.aar` (CPU + OpenCL + Hexagon NPU + mtmd, arm64-v8a, Snapdragon) | `android/app/libs/` and `implementation files('libs/llama-cpp-dart.aar')` in Gradle | Build artifacts yourself with: @@ -248,7 +248,7 @@ plan.md // milestone-by-milestone roadmap `0.9.x` is the rewrite line. The Dart API is mostly stable but **may break once more** before 1.0 — most likely around: real Jinja support, on-device validation findings, and final naming for chat-template/policy knobs. Pin to a minor when you ship. -llama.cpp is pinned per release in `src/llama.cpp` (git submodule). Bumps are tested against the full suite before tagging. The current pin is **tag `b9360`** (sha `6b4e4bd58`); if you're building your own native libs to match this package, check out that tag. +llama.cpp is pinned per release in `src/llama.cpp` (git submodule). Bumps are tested against the full suite before tagging. The current pin is **tag `b10182`** (sha `afeebe103`); if you're building your own native libs to match this package, check out that tag. ## License diff --git a/ios/Llama.xcframework/Info.plist b/ios/Llama.xcframework/Info.plist new file mode 100644 index 00000000..2a0bba71 --- /dev/null +++ b/ios/Llama.xcframework/Info.plist @@ -0,0 +1,58 @@ + + + + + AvailableLibraries + + + BinaryPath + Llama.framework/Llama + LibraryIdentifier + ios-arm64 + LibraryPath + Llama.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + BinaryPath + Llama.framework/Llama + LibraryIdentifier + ios-arm64_x86_64-simulator + LibraryPath + Llama.framework + SupportedArchitectures + + arm64 + x86_64 + + SupportedPlatform + ios + SupportedPlatformVariant + simulator + + + BinaryPath + Llama.framework/Versions/A/Llama + LibraryIdentifier + macos-arm64 + LibraryPath + Llama.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + macos + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-alloc.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-alloc.h new file mode 100644 index 00000000..a7926a21 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-alloc.h @@ -0,0 +1,86 @@ +#pragma once + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; +typedef struct ggml_backend_buffer * ggml_backend_buffer_t; +typedef struct ggml_backend * ggml_backend_t; + +// Tensor allocator +struct ggml_tallocr { + ggml_backend_buffer_t buffer; + void * base; + size_t alignment; + size_t offset; +}; + +GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer); +GGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor); + +// Graph allocator +/* + Example usage: + ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + + // optional: create a worst-case graph and reserve the buffers to avoid reallocations + ggml_gallocr_reserve(galloc, build_graph(max_batch)); + + // allocate the graph + struct ggml_cgraph * graph = build_graph(batch); + ggml_gallocr_alloc_graph(galloc, graph); + + printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0)); + + // evaluate the graph + ggml_backend_graph_compute(backend, graph); +*/ + +// special tensor flags for use with the graph allocator: +// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses +// ggml_set_output(): output tensors are never freed and never overwritten + +typedef struct ggml_gallocr * ggml_gallocr_t; + +GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft); +GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs); +GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc); + +// pre-allocate buffers from a measure graph - does not allocate or modify the graph +// call with a worst-case graph to avoid buffer reallocations +// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed +// returns false if the buffer allocation failed +// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes +GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph); +GGML_API void ggml_gallocr_reserve_n_size( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids, + size_t * sizes); +GGML_API bool ggml_gallocr_reserve_n( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids); + +// automatic reallocation if the topology changes when using a single buffer +// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers) +GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph); + +GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id); + +// Utils +// Create a buffer and allocate all the tensors in a ggml_context +// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft +// ggml_backend_alloc_ctx_tensors_from_buft returns NULL on failure or if all tensors in ctx are already allocated or zero-sized +GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-backend.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-backend.h new file mode 100644 index 00000000..2924fdbe --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-backend.h @@ -0,0 +1,435 @@ +#pragma once + +#include "ggml.h" +#include "ggml-alloc.h" + +#ifdef GGML_BACKEND_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BACKEND_BUILD +# define GGML_BACKEND_API __declspec(dllexport) extern +# else +# define GGML_BACKEND_API __declspec(dllimport) extern +# endif +# else +# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_BACKEND_API extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; + typedef struct ggml_backend_buffer * ggml_backend_buffer_t; + typedef struct ggml_backend_event * ggml_backend_event_t; + typedef struct ggml_backend * ggml_backend_t; + typedef void * ggml_backend_graph_plan_t; + typedef struct ggml_backend_reg * ggml_backend_reg_t; + typedef struct ggml_backend_device * ggml_backend_dev_t; + + + // + // Backend buffer type + // + + GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); + GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); + GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft); + + // + // Backend buffer + // + + enum ggml_backend_buffer_usage { + GGML_BACKEND_BUFFER_USAGE_ANY = 0, + GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1, + GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2, + }; + + GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); + GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); + GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor); + GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); + GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer); + GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); + + // tensor copy between different backends + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); + + // + // Backend (stream) + // + + GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend); + GGML_API const char * ggml_backend_name(ggml_backend_t backend); + GGML_API void ggml_backend_free(ggml_backend_t backend); + + GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend); + GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size); + GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); + GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); + + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + + // "offset" refers to the offset in tensor->data for setting/getting data + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + + GGML_API void ggml_backend_synchronize(ggml_backend_t backend); + + GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + + GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); + + // NOTE: will be removed, use device version instead + GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); + GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op); + + // asynchronous copy + // the copy is performed after all the currently queued operations in backend_src + // backend_dst will wait for the copy to complete before performing other operations + // automatic fallback to sync copy if async is not supported + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); + + GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); + + // + // Events + // + + GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device); + GGML_API void ggml_backend_event_free(ggml_backend_event_t event); + GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend); + GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event); + GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event); + + // + // Backend device + // + + enum ggml_backend_dev_type { + // CPU device using system memory + GGML_BACKEND_DEVICE_TYPE_CPU, + // GPU device using dedicated memory + GGML_BACKEND_DEVICE_TYPE_GPU, + // integrated GPU device using host memory + GGML_BACKEND_DEVICE_TYPE_IGPU, + // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, + }; + + // functionality supported by the device + struct ggml_backend_dev_caps { + // asynchronous operations + bool async; + // pinned host buffer + bool host_buffer; + // creating buffers from host ptr + bool buffer_from_host_ptr; + // event synchronization + bool events; + }; + + // all the device properties + struct ggml_backend_dev_props { + // device name + const char * name; + // device description + const char * description; + // device free memory in bytes + size_t memory_free; + // device total memory in bytes + size_t memory_total; + // device type + enum ggml_backend_dev_type type; + // device id + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") + // if the id is unknown, this should be NULL + const char * device_id; + // device capabilities + struct ggml_backend_dev_caps caps; + }; + + GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device); + GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total); + GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props); + GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device); + GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size); + + GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + + // + // Backend (reg) + // + + GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg); + GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg); + GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index); + GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name); + + // Common functions that may be obtained using ggml_backend_reg_get_proc_address + + // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) + typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); + typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); + + // Split buffer type for tensor parallelism (old) + typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); + // Set the number of threads for the backend + typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); + // Get additional buffer types provided by the device (returns a NULL-terminated array) + typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device); + // Set the abort callback for the backend + typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data); + // Get a list of feature flags supported by the backend (returns a NULL-terminated array) + struct ggml_backend_feature { + const char * name; + const char * value; + }; + typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg); + + // + // Backend registry + // + + GGML_API void ggml_backend_register(ggml_backend_reg_t reg); + + GGML_API void ggml_backend_device_register(ggml_backend_dev_t device); + + // Backend (reg) enumeration + GGML_API size_t ggml_backend_reg_count(void); + GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index); + GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name); + + // Device enumeration + GGML_API size_t ggml_backend_dev_count(void); + GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type); + + // Direct backend (stream) initialization + // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params) + GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params) + GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL) + GGML_API ggml_backend_t ggml_backend_init_best(void); + + // Load a backend from a dynamic library and register it + GGML_API ggml_backend_reg_t ggml_backend_load(const char * path); + // Unload a backend if loaded dynamically and unregister it + GGML_API void ggml_backend_unload(ggml_backend_reg_t reg); + // Load all known backends from dynamic libraries + GGML_API void ggml_backend_load_all(void); + GGML_API void ggml_backend_load_all_from_path(const char * dir_path); + + // + // Backend scheduler + // + + // The backend scheduler allows for multiple backend devices to be used together + // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends + // The backends are selected based on: + // - the backend that supports the operation + // - the location of the pre-allocated tensors (e.g. the weights) + /* + Example usage: + + // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned + // preferably to run on the same backend as the buffer + ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true); + + // initialize buffers from a max size graph (optional) + reserve_graph = build_graph(sched, max_batch_size); + + // manually assign nodes to a backend (optional, should not be needed in most cases) + struct ggml_tensor * node = ggml_mul_mat(ctx, ...); + ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu); + + ggml_backend_sched_reserve(sched, reserve_graph); + + // compute + graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation + for (int i = 0; i < 10; ++i) { + ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically + } + + // if there are graph inputs: + graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called) + ggml_backend_sched_reset(sched); // clear the allocation of the previous graph + ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it + ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors + ggml_backend_sched_graph_compute(sched, graph); // execute the graph + + // as an alternative to the above it is also possible to assign the inputs to a dedicated context and + // allocate them statically via ggml_backend_alloc_ctx_tensors + } + */ + + typedef struct ggml_backend_sched * ggml_backend_sched_t; + + // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback) + // when ask == true, the scheduler wants to know if the user wants to observe this node + // this allows the scheduler to batch nodes together in order to evaluate them in a single call + // + // when ask == false, the scheduler is passing the node tensor to the user for observation + // if the user returns false, the scheduler will cancel the graph compute + // + typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); + + // Initialize a backend scheduler, backends with low index are given priority over backends with high index + GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload); + GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); + + // Initialize backend buffers from a measure graph + GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes); + GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success + + GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched); + GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i); + + // Get the number of splits of the last graph + GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); + GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched); + + GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend); + GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + + GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); + + // Split graph without allocating it + GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + + // Allocate and compute graph on the backend scheduler + GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success + GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched); + + // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph. + // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers. + // The correct way to use this API is to discard the deallocated tensors and create new ones. + GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); + + // Set a callback to be called for each resulting node during graph compute + GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + + // + // Meta backend + // + +#define GGML_BACKEND_META_MAX_DEVICES 16 + + enum ggml_backend_meta_split_axis { + // tensor split by tensor dimensions: + GGML_BACKEND_SPLIT_AXIS_0 = 0, + GGML_BACKEND_SPLIT_AXIS_1 = 1, + GGML_BACKEND_SPLIT_AXIS_2 = 2, + GGML_BACKEND_SPLIT_AXIS_3 = 3, + + GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends + GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum + + // for internal bookkeeping only: + GGML_BACKEND_SPLIT_AXIS_NONE = 98, + GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99, + }; + GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis); + + struct ggml_backend_meta_split_state { + enum ggml_backend_meta_split_axis axis; + + // for tensors with axis >= 0 && axis < GGML_MAX_DIMS: + // - each device has a slice of the tensor along the split axis + // - most tensors have n_segments == 1 and a contiguous slice of the tensor data + // - some tensors have an inhomogenenous data layout along the split axis, + // those tensors are divided into segments which are each individually split across devices + // - ne has one entry per segment and device and that segment repeats nr times, + // in total when accounting for repetitions the segments add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0_r0, seg0_dev1_r0, seg0_dev0_r1, seg0_dev1_r1, seg1_dev0_r0, seg1_dev1_r0], + // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V, + // the Q matrix can be larger than the K and V matrices so this can either be expressed as 3 segments or as 2 segments + // where the segment for K/V repeats twice + int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t nr[16]; + uint32_t n_segments; + }; + + // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible: + typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata); + + // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this: + // TODO: this looks a bit strange - a backend API creates a device. I think we should try + // express this as a backend registry functionality instead + GGML_API ggml_backend_dev_t ggml_backend_meta_device( + ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud); + + // + // Utils + // + + struct ggml_backend_graph_copy { + ggml_backend_buffer_t buffer; + struct ggml_context * ctx_allocated; + struct ggml_context * ctx_unallocated; + struct ggml_cgraph * graph; + }; + + // Copy a graph to a different backend + GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph); + GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy); + + typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data); + + // Compare the output of two backends + GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes); + + // Tensor initialization + GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr); + GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor); + + // CPU buffer types are always available + GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); + GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-cpu.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-cpu.h new file mode 100644 index 00000000..e3e067c9 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-cpu.h @@ -0,0 +1,151 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + + // the compute plan that needs to be prepared for ggml_graph_compute() + // since https://github.com/ggml-org/ggml/issues/287 + struct ggml_cplan { + size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()` + uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()` + + int n_threads; + struct ggml_threadpool * threadpool; + + // abort ggml_graph_compute when true + ggml_abort_callback abort_callback; + void * abort_callback_data; + + // use only reference implementations + bool use_ref; + }; + + // numa strategies + enum ggml_numa_strategy { + GGML_NUMA_STRATEGY_DISABLED = 0, + GGML_NUMA_STRATEGY_DISTRIBUTE = 1, + GGML_NUMA_STRATEGY_ISOLATE = 2, + GGML_NUMA_STRATEGY_NUMACTL = 3, + GGML_NUMA_STRATEGY_MIRROR = 4, + GGML_NUMA_STRATEGY_COUNT + }; + + GGML_BACKEND_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems + GGML_BACKEND_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node + + GGML_BACKEND_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value); + GGML_BACKEND_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value); + + GGML_BACKEND_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value); + GGML_BACKEND_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value); + + GGML_BACKEND_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i); + GGML_BACKEND_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value); + + GGML_BACKEND_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3); + GGML_BACKEND_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value); + + GGML_BACKEND_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i); + GGML_BACKEND_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value); + + GGML_BACKEND_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3); + GGML_BACKEND_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value); + + GGML_BACKEND_API struct ggml_threadpool * ggml_threadpool_new (struct ggml_threadpool_params * params); + GGML_BACKEND_API void ggml_threadpool_free (struct ggml_threadpool * threadpool); + GGML_BACKEND_API int ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool); + GGML_BACKEND_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool); + GGML_BACKEND_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool); + + // ggml_graph_plan() has to be called before ggml_graph_compute() + // when plan.work_size > 0, caller must allocate memory for plan.work_data + GGML_BACKEND_API struct ggml_cplan ggml_graph_plan( + const struct ggml_cgraph * cgraph, + int n_threads, /* = GGML_DEFAULT_N_THREADS */ + struct ggml_threadpool * threadpool /* = NULL */ ); + GGML_BACKEND_API enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan); + + // same as ggml_graph_compute() but the work data is allocated as a part of the context + // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data + GGML_BACKEND_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads); + + // + // system info + // + + // x86 + GGML_BACKEND_API int ggml_cpu_has_sse3 (void); + GGML_BACKEND_API int ggml_cpu_has_ssse3 (void); + GGML_BACKEND_API int ggml_cpu_has_avx (void); + GGML_BACKEND_API int ggml_cpu_has_avx_vnni (void); + GGML_BACKEND_API int ggml_cpu_has_avx2 (void); + GGML_BACKEND_API int ggml_cpu_has_bmi2 (void); + GGML_BACKEND_API int ggml_cpu_has_f16c (void); + GGML_BACKEND_API int ggml_cpu_has_fma (void); + GGML_BACKEND_API int ggml_cpu_has_avx512 (void); + GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void); + GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void); + GGML_BACKEND_API int ggml_cpu_has_avx512_bf16(void); + GGML_BACKEND_API int ggml_cpu_has_amx_int8 (void); + // ARM + GGML_BACKEND_API int ggml_cpu_has_neon (void); + GGML_BACKEND_API int ggml_cpu_has_arm_fma (void); + GGML_BACKEND_API int ggml_cpu_has_fp16_va (void); + GGML_BACKEND_API int ggml_cpu_has_dotprod (void); + GGML_BACKEND_API int ggml_cpu_has_matmul_int8(void); + GGML_BACKEND_API int ggml_cpu_has_sve (void); + GGML_BACKEND_API int ggml_cpu_get_sve_cnt (void); // sve vector length in bytes + GGML_BACKEND_API int ggml_cpu_has_sme (void); + // other + GGML_BACKEND_API int ggml_cpu_has_riscv_v (void); + GGML_BACKEND_API int ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytes + GGML_BACKEND_API int ggml_cpu_has_vsx (void); + GGML_BACKEND_API int ggml_cpu_has_vxe (void); + GGML_BACKEND_API int ggml_cpu_has_wasm_simd (void); + GGML_BACKEND_API int ggml_cpu_has_llamafile (void); + + // Internal types and functions exposed for tests and benchmarks + + typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx, + const void * GGML_RESTRICT y, size_t by, int nrc); + + struct ggml_type_traits_cpu { + ggml_from_float_t from_float; + ggml_vec_dot_t vec_dot; + enum ggml_type vec_dot_type; + int64_t nrows; // number of rows to process simultaneously + }; + + GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type); + + GGML_BACKEND_API void ggml_cpu_init(void); + + // + // CPU backend + // + + GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void); + + GGML_BACKEND_API bool ggml_backend_is_cpu (ggml_backend_t backend); + GGML_BACKEND_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads); + GGML_BACKEND_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool); + GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data); + + GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref); + + GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void); + + GGML_BACKEND_API void ggml_cpu_fp32_to_fp32(const float *, float *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_i32 (const float *, int32_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_fp16(const float *, ggml_fp16_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp16_to_fp32(const ggml_fp16_t *, float *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_bf16(const float *, ggml_bf16_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_bf16_to_fp32(const ggml_bf16_t *, float *, int64_t); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-metal.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-metal.h new file mode 100644 index 00000000..433838f0 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-metal.h @@ -0,0 +1,61 @@ +// Note: this description is outdated +// +// An interface allowing to compute ggml_cgraph with Metal +// +// This is a fully functional interface that extends ggml with GPU support for Apple devices. +// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.) +// +// How it works? +// +// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this +// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you +// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.) +// +// You only need to make sure that all memory buffers that you used during the graph creation +// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is +// used during the graph evaluation to determine the arguments of the compute kernels. +// +// Synchronization between device and host memory (for example for input and output tensors) +// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions. +// + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include + +struct ggml_tensor; +struct ggml_cgraph; + +#ifdef __cplusplus +extern "C" { +#endif + +// +// backend API +// user-code should use only these functions +// + +// TODO: remove in the future +GGML_BACKEND_API ggml_backend_t ggml_backend_metal_init(void); + +GGML_BACKEND_API bool ggml_backend_is_metal(ggml_backend_t backend); + +GGML_BACKEND_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data); + +// helper to check if the device supports a specific family +// ideally, the user code should be doing these checks +// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf +GGML_BACKEND_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family); + +// capture all command buffers committed the next time `ggml_backend_graph_compute` is called +GGML_BACKEND_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_metal_reg(void); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-opt.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-opt.h new file mode 100644 index 00000000..1c2ed79b --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml-opt.h @@ -0,0 +1,256 @@ +// This file contains functionality for training models using GGML. +// It is not strictly needed vs. just vanilla GGML but it provides a more high-level interface for common needs such as datasets. +// At the bottom of this file especially there are relatively high-level functions that are suitable use or adaptation in user code. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + struct ggml_opt_dataset; + struct ggml_opt_context; + struct ggml_opt_result; + + typedef struct ggml_opt_dataset * ggml_opt_dataset_t; + typedef struct ggml_opt_context * ggml_opt_context_t; + typedef struct ggml_opt_result * ggml_opt_result_t; + + // ====== Loss ====== + + // built-in loss types, i.e. the built-in quantities minimized by the optimizer + // custom loss types can be defined via mean or sum which simply reduce the outputs for all datapoints to a single value + enum ggml_opt_loss_type { + GGML_OPT_LOSS_TYPE_MEAN, + GGML_OPT_LOSS_TYPE_SUM, + GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, + GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR, + }; + + // ====== Dataset ====== + + GGML_API ggml_opt_dataset_t ggml_opt_dataset_init( + enum ggml_type type_data, // the type for the internal data tensor + enum ggml_type type_label, // the type for the internal labels tensor + int64_t ne_datapoint, // number of elements per datapoint + int64_t ne_label, // number of elements per label + int64_t ndata, // total number of datapoints/labels + int64_t ndata_shard); // number of datapoints/labels per shard (unit at which the dataset is shuffled/copied) + GGML_API void ggml_opt_dataset_free(ggml_opt_dataset_t dataset); + + // get underlying tensors that store the data + GGML_API int64_t ggml_opt_dataset_ndata (ggml_opt_dataset_t dataset); + GGML_API struct ggml_tensor * ggml_opt_dataset_data (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata] + GGML_API struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset); // shape = [nd_label, ndata] + + // shuffle idata first datapoints from dataset with RNG from opt_ctx, shuffle all datapoints if idata is negative + GGML_API void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata); + + // get batch at position ibatch from dataset and copy the data to data_batch and labels_batch + GGML_API void ggml_opt_dataset_get_batch( + ggml_opt_dataset_t dataset, + struct ggml_tensor * data_batch, // shape = [ne_datapoint, ndata_batch] + struct ggml_tensor * labels_batch, // shape = [ne_label, ndata_batch] + int64_t ibatch); + GGML_API void ggml_opt_dataset_get_batch_host( + ggml_opt_dataset_t dataset, + void * data_batch, + size_t nb_data_batch, + void * labels_batch, + int64_t ibatch); + + // ====== Model / Context ====== + + enum ggml_opt_build_type { + GGML_OPT_BUILD_TYPE_FORWARD = 10, + GGML_OPT_BUILD_TYPE_GRAD = 20, + GGML_OPT_BUILD_TYPE_OPT = 30, + }; + + enum ggml_opt_optimizer_type { + GGML_OPT_OPTIMIZER_TYPE_ADAMW, + GGML_OPT_OPTIMIZER_TYPE_SGD, + + GGML_OPT_OPTIMIZER_TYPE_COUNT + }; + + // parameters that control which optimizer is used and how said optimizer tries to find the minimal loss + struct ggml_opt_optimizer_params { + struct { + float alpha; // learning rate + float beta1; // first AdamW momentum + float beta2; // second AdamW momentum + float eps; // epsilon for numerical stability + float wd; // weight decay - 0.0f to disable + } adamw; + struct { + float alpha; // learning rate + float wd; // weight decay + } sgd; + }; + + // callback to calculate optimizer parameters prior to a backward pass + // userdata can be used to pass arbitrary data + typedef struct ggml_opt_optimizer_params (*ggml_opt_get_optimizer_params)(void * userdata); + + // returns the default optimizer params (constant, hard-coded values) + // userdata is not used + GGML_API struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata); + + // casts userdata to ggml_opt_optimizer_params and returns it + GGML_API struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata); + + // parameters for initializing a new optimization context + struct ggml_opt_params { + ggml_backend_sched_t backend_sched; // defines which backends are used to construct the compute graphs + + // by default the forward graph needs to be reconstructed for each eval + // if ctx_compute, inputs, and outputs are set the graphs are instead allocated statically + struct ggml_context * ctx_compute; + struct ggml_tensor * inputs; + struct ggml_tensor * outputs; + + enum ggml_opt_loss_type loss_type; + enum ggml_opt_build_type build_type; + + int32_t opt_period; // after how many gradient accumulation steps an optimizer step should be done + + ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters + void * get_opt_pars_ud; // userdata for calculating optimizer parameters + + // only GGML_OPT_OPTIMIZER_TYPE_ADAMW needs m, v momenta per parameter tensor + enum ggml_opt_optimizer_type optimizer; + }; + + // get parameters for an optimization context with defaults set where possible + // parameters for which no sensible defaults exist are supplied as arguments to this function + GGML_API struct ggml_opt_params ggml_opt_default_params( + ggml_backend_sched_t backend_sched, + enum ggml_opt_loss_type loss_type); + + GGML_API ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params); + GGML_API void ggml_opt_free(ggml_opt_context_t opt_ctx); + + // set gradients to zero, initialize loss, and optionally reset the optimizer + GGML_API void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer); + + GGML_API bool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx); // whether the graphs are allocated_statically + + // get underlying tensors that store data + // if not using static graphs these pointers become invalid with the next call to ggml_opt_alloc + GGML_API struct ggml_tensor * ggml_opt_inputs( ggml_opt_context_t opt_ctx); // forward graph input tensor + GGML_API struct ggml_tensor * ggml_opt_outputs( ggml_opt_context_t opt_ctx); // forward graph output tensor + GGML_API struct ggml_tensor * ggml_opt_labels( ggml_opt_context_t opt_ctx); // labels to compare outputs against + GGML_API struct ggml_tensor * ggml_opt_loss( ggml_opt_context_t opt_ctx); // scalar tensor that contains the loss + GGML_API struct ggml_tensor * ggml_opt_pred( ggml_opt_context_t opt_ctx); // predictions made by outputs + GGML_API struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx); // number of matching predictions between outputs and labels + + // get the gradient accumulator for a node from the forward graph + GGML_API struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node); + + GGML_API enum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t); //TODO consistent naming scheme + + GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type); + + // ====== Optimization Result ====== + + GGML_API ggml_opt_result_t ggml_opt_result_init(void); + GGML_API void ggml_opt_result_free(ggml_opt_result_t result); + GGML_API void ggml_opt_result_reset(ggml_opt_result_t result); + + // get data from result, uncertainties are optional and can be ignored by passing NULL + GGML_API void ggml_opt_result_ndata( ggml_opt_result_t result, int64_t * ndata); // writes 1 value, number of datapoints + GGML_API void ggml_opt_result_loss( ggml_opt_result_t result, double * loss, double * unc); // writes 1 value + GGML_API void ggml_opt_result_pred( ggml_opt_result_t result, int32_t * pred); // writes ndata values + GGML_API void ggml_opt_result_accuracy(ggml_opt_result_t result, double * accuracy, double * unc); // writes 1 value + + // ====== Computation ====== + + // if not using static graphs, this function must be called prior to ggml_opt_alloc + GGML_API void ggml_opt_prepare_alloc( + ggml_opt_context_t opt_ctx, + struct ggml_context * ctx_compute, + struct ggml_cgraph * gf, + struct ggml_tensor * inputs, + struct ggml_tensor * outputs); + + // allocate the next graph for evaluation, either forward or forward + backward + // must be called exactly once prior to calling ggml_opt_eval + GGML_API void ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward); + + // do forward pass, increment result if not NULL, do backward pass if allocated + GGML_API void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result); + + // ############################################################################ + // ## The high-level functions start here. They do not depend on any private ## + // ## functions or structs and can be copied to and adapted for user code. ## + // ############################################################################ + + // ====== Intended Usage ====== + // + // 1. Select the appropriate loss for your problem. + // 2. Create a dataset and set the data for the "data" tensor. Also set the "labels" tensor if your loss needs them. + // Setting the shard size to 1 will be fine, it's the granularity with which data is shuffled/loaded (bigger values are faster). + // 3. Create a GGML graph for your model with no_alloc == true. Use two separate contexts for the tensors. + // The first context should contain the model parameters and inputs and be allocated statically in user code. + // The second context should contain all other tensors and will be (re)allocated automatically. + // Due to this automated allocation the data of the second context is not defined when accessed in user code. + // Note that the second dimension of the inputs/outputs are interpreted as the number of datapoints in those tensors. + // 4. Call ggml_opt_fit. If you need more control you can use ggml_opt_epoch instead. + + // signature for a callback while evaluating opt_ctx on dataset, called after an evaluation + typedef void (*ggml_opt_epoch_callback)( + bool train, // true after training evaluation, false after validation evaluation + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, // result associated with the dataset subsection + int64_t ibatch, // number of batches that have been evaluated so far + int64_t ibatch_max, // total number of batches in this dataset subsection + int64_t t_start_us); // time at which the evaluation on the dataset subsection was started + + // do training on front of dataset, do evaluation only on back of dataset + GGML_API void ggml_opt_epoch( + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, // result to increment during training, ignored if NULL + ggml_opt_result_t result_eval, // result to increment during evaluation, ignored if NULL + int64_t idata_split, // data index at which to split training and evaluation + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval); + + // callback that prints a progress bar on stderr + GGML_API void ggml_opt_epoch_callback_progress_bar( + bool train, + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, + int64_t ibatch, + int64_t ibatch_max, + int64_t t_start_us); + + // fit model defined by inputs and outputs to dataset + GGML_API void ggml_opt_fit( + ggml_backend_sched_t backend_sched, // backend scheduler for constructing the compute graphs + struct ggml_context * ctx_compute, // context with temporarily allocated tensors to calculate the outputs + struct ggml_tensor * inputs, // input tensor with shape [ne_datapoint, ndata_batch] + struct ggml_tensor * outputs, // output tensor, must have shape [ne_label, ndata_batch] if labels are used + ggml_opt_dataset_t dataset, // dataset with data and optionally also labels + enum ggml_opt_loss_type loss_type, // loss to minimize + enum ggml_opt_optimizer_type optimizer, // sgd or adamw + ggml_opt_get_optimizer_params get_opt_pars, // callback to get optimizer params, userdata is pointer to epoch (of type int64_t) + int64_t nepoch, // how many times the dataset should be iterated over + int64_t nbatch_logical, // datapoints optimizer step, must be a multiple of ndata_batch in inputs/outputs + float val_split, // fraction of the dataset to use for validation, must be in [0.0f, 1.0f) + bool silent); // whether or not info prints to stderr should be suppressed + + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml.h new file mode 100644 index 00000000..374934aa --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/ggml.h @@ -0,0 +1,2856 @@ +#pragma once + +// +// GGML Tensor Library +// +// This documentation is still a work in progress. +// If you wish some specific topics to be covered, feel free to drop a comment: +// +// https://github.com/ggml-org/whisper.cpp/issues/40 +// +// ## Overview +// +// This library implements: +// +// - a set of tensor operations +// - automatic differentiation +// - basic optimization algorithms +// +// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes, +// but is not limited to, the following: +// +// - linear regression +// - support vector machines +// - neural networks +// +// The library allows the user to define a certain function using the available tensor operations. This function +// definition is represented internally via a computation graph. Each tensor operation in the function definition +// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the +// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized +// using one of the available optimization algorithms. +// +// For example, here we define the function: f(x) = a*x^2 + b +// +// { +// struct ggml_init_params params = { +// .mem_size = 16*1024*1024, +// .mem_buffer = NULL, +// }; +// +// // memory allocation happens here +// struct ggml_context * ctx = ggml_init(params); +// +// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// +// ggml_set_param(ctx, x); // x is an input variable +// +// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * x2 = ggml_mul(ctx, x, x); +// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b); +// +// ... +// } +// +// Notice that the function definition above does not involve any actual computation. The computation is performed only +// when the user explicitly requests it. For example, to compute the function's value at x = 2.0: +// +// { +// ... +// +// struct ggml_cgraph * gf = ggml_new_graph(ctx); +// ggml_build_forward_expand(gf, f); +// +// // set the input variable and parameter values +// ggml_set_f32(x, 2.0f); +// ggml_set_f32(a, 3.0f); +// ggml_set_f32(b, 4.0f); +// +// ggml_graph_compute_with_ctx(ctx, &gf, n_threads); +// +// printf("f = %f\n", ggml_get_f32_1d(f, 0)); +// +// ... +// } +// +// The actual computation is performed in the ggml_graph_compute() function. +// +// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the +// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know +// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory +// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was +// actually needed. +// +// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic +// differentiation and optimization algorithms. +// +// The described approach allows to define the function graph once and then compute its forward or backward graphs +// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way +// the user can avoid the memory allocation overhead at runtime. +// +// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class +// citizens, but in theory the library can be extended to support FP8 and integer data types. +// +// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary +// and binary operations. Most of the available operations fall into one of these two categories. With time, it became +// clear that the library needs to support more complex operations. The way to support these operations is not clear +// yet, but a few examples are demonstrated in the following operations: +// +// - ggml_permute() +// - ggml_conv_1d_1s() +// - ggml_conv_1d_2s() +// +// For each tensor operator, the library implements a forward and backward computation function. The forward function +// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the +// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a +// calculus class, or watch the following video: +// +// What is Automatic Differentiation? +// https://www.youtube.com/watch?v=wG_nF1awSSY +// +// +// ## Tensor data (struct ggml_tensor) +// +// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of +// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains +// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example: +// +// { +// struct ggml_tensor * c = ggml_add(ctx, a, b); +// +// assert(c->src[0] == a); +// assert(c->src[1] == b); +// } +// +// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the +// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows +// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and +// permutation. All tensor operations have to take the stride into account and not assume that the tensor is +// contiguous in memory. +// +// The data of the tensor is accessed via the "data" pointer. For example: +// +// { +// const int nx = 2; +// const int ny = 3; +// +// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny); +// +// for (int y = 0; y < ny; y++) { +// for (int x = 0; x < nx; x++) { +// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y; +// } +// } +// +// ... +// } +// +// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used. +// +// ## The matrix multiplication operator (ggml_mul_mat) +// +// TODO +// +// +// ## Multi-threading +// +// TODO +// +// +// ## Overview of ggml.c +// +// TODO +// +// +// ## SIMD optimizations +// +// TODO +// +// +// ## Debugging ggml +// +// TODO +// +// + +#ifdef GGML_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BUILD +# define GGML_API __declspec(dllexport) extern +# else +# define GGML_API __declspec(dllimport) extern +# endif +# else +# define GGML_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_API extern +#endif + +// TODO: support for clang +#ifdef __GNUC__ +# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define GGML_DEPRECATED(func, hint) func +#endif + +#ifndef __GNUC__ +# define GGML_ATTRIBUTE_FORMAT(...) +#elif defined(__MINGW32__) && !defined(__clang__) +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__))) +#else +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__))) +#endif + +#if defined(_WIN32) && !defined(_WIN32_WINNT) +# define _WIN32_WINNT 0x0A00 +#endif + +#include +#include +#include +#include + +#define GGML_FILE_MAGIC 0x67676d6c // "ggml" +#define GGML_FILE_VERSION 2 + +#define GGML_QNT_VERSION 2 // bump this on quantization format changes +#define GGML_QNT_VERSION_FACTOR 1000 // do not change this + +#define GGML_MAX_DIMS 4 +#define GGML_MAX_PARAMS 2048 +#define GGML_MAX_SRC 10 +#define GGML_MAX_N_THREADS 512 +#define GGML_MAX_OP_PARAMS 64 + +#ifndef GGML_MAX_NAME +# define GGML_MAX_NAME 64 +#endif + +#define GGML_DEFAULT_N_THREADS 4 +#define GGML_DEFAULT_GRAPH_SIZE 2048 + +#if UINTPTR_MAX == 0xFFFFFFFF + #define GGML_MEM_ALIGN 4 +#elif defined(__EMSCRIPTEN__) +// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm. +// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.) +// ref: https://github.com/ggml-org/llama.cpp/pull/18628 + #define GGML_MEM_ALIGN 8 +#else + #define GGML_MEM_ALIGN 16 +#endif + +#define GGML_EXIT_SUCCESS 0 +#define GGML_EXIT_ABORTED 1 + +// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726 +#define GGML_ROPE_TYPE_NORMAL 0 +#define GGML_ROPE_TYPE_NEOX 2 +#define GGML_ROPE_TYPE_MROPE 8 +#define GGML_ROPE_TYPE_VISION 24 +#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000 + +#define GGML_MROPE_SECTIONS 4 + +#define GGML_UNUSED(x) (void)(x) +#ifdef __CUDACC__ +template +__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {} +#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__) +#else +#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0) +#endif // __CUDACC__ + +#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) + +#ifndef NDEBUG +# define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0) +#elif defined(__GNUC__) +# define GGML_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) +# define GGML_UNREACHABLE() __assume(0) +#else +# define GGML_UNREACHABLE() ((void) 0) +#endif + +#ifdef __cplusplus +# define GGML_NORETURN [[noreturn]] +#elif defined(_MSC_VER) +# define GGML_NORETURN __declspec(noreturn) +#else +# define GGML_NORETURN _Noreturn +#endif + +#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__) +#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x) + +// used to copy the number of elements and stride in bytes of tensors into local variables. +// main purpose is to reduce code duplication and improve readability. +// +// example: +// +// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); +// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb); +// +#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \ + const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \ + GGML_UNUSED(prefix##0); +#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \ + const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \ + GGML_UNUSED(prefix##1); +#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \ + const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \ + GGML_UNUSED(prefix##2); +#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \ + const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \ + GGML_UNUSED(prefix##3); + +#define GGML_TENSOR_UNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_TERNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \ + GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS01 \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) + +#ifdef __cplusplus +extern "C" { +#endif + + // Function type used in fatal error callbacks + typedef void (*ggml_abort_callback_t)(const char * error_message); + + // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout) + // Returns the old callback for chaining + GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback); + + GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4) + GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...); + + enum ggml_status { + GGML_STATUS_ALLOC_FAILED = -2, + GGML_STATUS_FAILED = -1, + GGML_STATUS_SUCCESS = 0, + GGML_STATUS_ABORTED = 1, + }; + + // get ggml_status name string + GGML_API const char * ggml_status_to_string(enum ggml_status status); + + // ieee 754-2008 half-precision float16 + // todo: make this not an integral type + typedef uint16_t ggml_fp16_t; + GGML_API float ggml_fp16_to_fp32(ggml_fp16_t); + GGML_API ggml_fp16_t ggml_fp32_to_fp16(float); + GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t); + + // google brain half-precision bfloat16 + typedef struct { uint16_t bits; } ggml_bf16_t; + GGML_API ggml_bf16_t ggml_fp32_to_bf16(float); + GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16 + GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t); + GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t); + + struct ggml_object; + struct ggml_context; + struct ggml_cgraph; + + // NOTE: always add types at the end of the enum to keep backward compatibility + enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + // GGML_TYPE_Q4_2 = 4, support has been removed + // GGML_TYPE_Q4_3 = 5, support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, + GGML_TYPE_IQ2_XXS = 16, + GGML_TYPE_IQ2_XS = 17, + GGML_TYPE_IQ3_XXS = 18, + GGML_TYPE_IQ1_S = 19, + GGML_TYPE_IQ4_NL = 20, + GGML_TYPE_IQ3_S = 21, + GGML_TYPE_IQ2_S = 22, + GGML_TYPE_IQ4_XS = 23, + GGML_TYPE_I8 = 24, + GGML_TYPE_I16 = 25, + GGML_TYPE_I32 = 26, + GGML_TYPE_I64 = 27, + GGML_TYPE_F64 = 28, + GGML_TYPE_IQ1_M = 29, + GGML_TYPE_BF16 = 30, + // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // GGML_TYPE_Q4_0_4_8 = 32, + // GGML_TYPE_Q4_0_8_8 = 33, + GGML_TYPE_TQ1_0 = 34, + GGML_TYPE_TQ2_0 = 35, + // GGML_TYPE_IQ4_NL_4_4 = 36, + // GGML_TYPE_IQ4_NL_4_8 = 37, + // GGML_TYPE_IQ4_NL_8_8 = 38, + GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) + GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_COUNT = 42, + }; + + // precision + enum ggml_prec { + GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default + GGML_PREC_F32 = 10, + }; + + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, + }; + + // model file types + enum ggml_ftype { + GGML_FTYPE_UNKNOWN = -1, + GGML_FTYPE_ALL_F32 = 0, + GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors + GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors + GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors + GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors + GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + }; + + // available tensor operations: + enum ggml_op { + GGML_OP_NONE = 0, + + GGML_OP_DUP, + GGML_OP_ADD, + GGML_OP_ADD_ID, + GGML_OP_ADD1, + GGML_OP_ACC, + GGML_OP_SUB, + GGML_OP_MUL, + GGML_OP_DIV, + GGML_OP_SQR, + GGML_OP_SQRT, + GGML_OP_LOG, + GGML_OP_SIN, + GGML_OP_COS, + GGML_OP_SUM, + GGML_OP_SUM_ROWS, + GGML_OP_CUMSUM, + GGML_OP_MEAN, + GGML_OP_ARGMAX, + GGML_OP_COUNT_EQUAL, + GGML_OP_REPEAT, + GGML_OP_REPEAT_BACK, + GGML_OP_CONCAT, + GGML_OP_SILU_BACK, + GGML_OP_NORM, // normalize + GGML_OP_RMS_NORM, + GGML_OP_RMS_NORM_BACK, + GGML_OP_GROUP_NORM, + GGML_OP_L2_NORM, + + GGML_OP_MUL_MAT, + GGML_OP_MUL_MAT_ID, + GGML_OP_OUT_PROD, + + GGML_OP_SCALE, + GGML_OP_SET, + GGML_OP_CPY, + GGML_OP_CONT, + GGML_OP_RESHAPE, + GGML_OP_VIEW, + GGML_OP_PERMUTE, + GGML_OP_TRANSPOSE, + GGML_OP_GET_ROWS, + GGML_OP_GET_ROWS_BACK, + GGML_OP_SET_ROWS, + GGML_OP_DIAG, + GGML_OP_DIAG_MASK_INF, + GGML_OP_DIAG_MASK_ZERO, + GGML_OP_SOFT_MAX, + GGML_OP_SOFT_MAX_BACK, + GGML_OP_ROPE, + GGML_OP_ROPE_BACK, + GGML_OP_CLAMP, + GGML_OP_CONV_TRANSPOSE_1D, + GGML_OP_IM2COL, + GGML_OP_IM2COL_BACK, + GGML_OP_IM2COL_3D, + GGML_OP_COL2IM_1D, + GGML_OP_CONV_2D, + GGML_OP_CONV_3D, + GGML_OP_CONV_2D_DW, + GGML_OP_CONV_TRANSPOSE_2D, + GGML_OP_POOL_1D, + GGML_OP_POOL_2D, + GGML_OP_POOL_2D_BACK, + GGML_OP_UPSCALE, + GGML_OP_PAD, + GGML_OP_PAD_REFLECT_1D, + GGML_OP_ROLL, + GGML_OP_ARANGE, + GGML_OP_TIMESTEP_EMBEDDING, + GGML_OP_ARGSORT, + GGML_OP_TOP_K, + GGML_OP_LEAKY_RELU, + GGML_OP_TRI, + GGML_OP_FILL, + + GGML_OP_FLASH_ATTN_EXT, + GGML_OP_FLASH_ATTN_BACK, + GGML_OP_SSM_CONV, + GGML_OP_SSM_SCAN, + GGML_OP_WIN_PART, + GGML_OP_WIN_UNPART, + GGML_OP_GET_REL_POS, + GGML_OP_ADD_REL_POS, + GGML_OP_RWKV_WKV6, + GGML_OP_GATED_LINEAR_ATTN, + GGML_OP_RWKV_WKV7, + GGML_OP_SOLVE_TRI, + GGML_OP_GATED_DELTA_NET, + + GGML_OP_UNARY, + + GGML_OP_MAP_CUSTOM1, + GGML_OP_MAP_CUSTOM2, + GGML_OP_MAP_CUSTOM3, + + GGML_OP_CUSTOM, + + GGML_OP_CROSS_ENTROPY_LOSS, + GGML_OP_CROSS_ENTROPY_LOSS_BACK, + GGML_OP_OPT_STEP_ADAMW, + GGML_OP_OPT_STEP_SGD, + + GGML_OP_GLU, + + GGML_OP_COUNT, + }; + + enum ggml_unary_op { + GGML_UNARY_OP_ABS, + GGML_UNARY_OP_SGN, + GGML_UNARY_OP_NEG, + GGML_UNARY_OP_STEP, + GGML_UNARY_OP_TANH, + GGML_UNARY_OP_ELU, + GGML_UNARY_OP_RELU, + GGML_UNARY_OP_SIGMOID, + GGML_UNARY_OP_GELU, + GGML_UNARY_OP_GELU_QUICK, + GGML_UNARY_OP_SILU, + GGML_UNARY_OP_HARDSWISH, + GGML_UNARY_OP_HARDSIGMOID, + GGML_UNARY_OP_EXP, + GGML_UNARY_OP_EXPM1, + GGML_UNARY_OP_SOFTPLUS, + GGML_UNARY_OP_GELU_ERF, + GGML_UNARY_OP_XIELU, + GGML_UNARY_OP_FLOOR, + GGML_UNARY_OP_CEIL, + GGML_UNARY_OP_ROUND, + GGML_UNARY_OP_TRUNC, + + GGML_UNARY_OP_COUNT, + }; + + enum ggml_glu_op { + GGML_GLU_OP_REGLU, + GGML_GLU_OP_GEGLU, + GGML_GLU_OP_SWIGLU, + GGML_GLU_OP_SWIGLU_OAI, + GGML_GLU_OP_GEGLU_ERF, + GGML_GLU_OP_GEGLU_QUICK, + + GGML_GLU_OP_COUNT, + }; + + enum ggml_object_type { + GGML_OBJECT_TYPE_TENSOR, + GGML_OBJECT_TYPE_GRAPH, + GGML_OBJECT_TYPE_WORK_BUFFER + }; + + enum ggml_log_level { + GGML_LOG_LEVEL_NONE = 0, + GGML_LOG_LEVEL_DEBUG = 1, + GGML_LOG_LEVEL_INFO = 2, + GGML_LOG_LEVEL_WARN = 3, + GGML_LOG_LEVEL_ERROR = 4, + GGML_LOG_LEVEL_CONT = 5, // continue previous log + }; + + // this tensor... + enum ggml_tensor_flag { + GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph + GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph + GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters + GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up) + GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed + }; + + enum ggml_tri_type { + GGML_TRI_TYPE_UPPER_DIAG = 0, + GGML_TRI_TYPE_UPPER = 1, + GGML_TRI_TYPE_LOWER_DIAG = 2, + GGML_TRI_TYPE_LOWER = 3 + }; + + struct ggml_init_params { + // memory pool + size_t mem_size; // bytes + void * mem_buffer; // if NULL, memory will be allocated internally + bool no_alloc; // don't allocate memory for the tensor data + }; + + // n-dimensional tensor + struct ggml_tensor { + enum ggml_type type; + + struct ggml_backend_buffer * buffer; + + int64_t ne[GGML_MAX_DIMS]; // number of elements + size_t nb[GGML_MAX_DIMS]; // stride in bytes: + // nb[0] = ggml_type_size(type) + // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding + // nb[i] = nb[i-1] * ne[i-1] + + // compute data + enum ggml_op op; + + // op params - allocated as int32_t for alignment + int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; + + int32_t flags; + + struct ggml_tensor * src[GGML_MAX_SRC]; + + // source tensor and offset for views + struct ggml_tensor * view_src; + size_t view_offs; + + void * data; + + char name[GGML_MAX_NAME]; + + void * extra; // extra things e.g. for ggml-cuda.cu + + char padding[8]; + }; + + static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); + + // Abort callback + // If not NULL, called before ggml computation + // If it returns true, the computation is aborted + typedef bool (*ggml_abort_callback)(void * data); + + + // + // GUID + // + + // GUID types + typedef uint8_t ggml_guid[16]; + typedef ggml_guid * ggml_guid_t; + + GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b); + + // misc + + GGML_API const char * ggml_version(void); + GGML_API const char * ggml_commit(void); + + GGML_API void ggml_time_init(void); // call this once at the beginning of the program + GGML_API int64_t ggml_time_ms(void); + GGML_API int64_t ggml_time_us(void); + GGML_API int64_t ggml_cycles(void); + GGML_API int64_t ggml_cycles_per_ms(void); + + // accepts a UTF-8 path, even on Windows + GGML_API FILE * ggml_fopen(const char * fname, const char * mode); + + GGML_API void ggml_print_object (const struct ggml_object * obj); + GGML_API void ggml_print_objects(const struct ggml_context * ctx); + + GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor); + GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN + + GGML_API int64_t ggml_blck_size(enum ggml_type type); + GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block + GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row + + GGML_DEPRECATED( + GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float + "use ggml_row_size() instead"); + + GGML_API const char * ggml_type_name(enum ggml_type type); + GGML_API const char * ggml_op_name (enum ggml_op op); + GGML_API const char * ggml_op_symbol(enum ggml_op op); + + GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op); + GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op); + GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name + + GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor); + + GGML_API bool ggml_is_quantized(enum ggml_type type); + + // TODO: temporary until model loading of ggml examples is refactored + GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype); + + GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor); + GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_view (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor); + GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars + + // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation) + GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous() + GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1 + GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2 + + // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok) + GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor); + + // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN + GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor); + + // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements + GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor); + + GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1); + GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + // use this to compute the memory overhead of a tensor + GGML_API size_t ggml_tensor_overhead(void); + + GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes); + + // main + + GGML_API struct ggml_context * ggml_init (struct ggml_init_params params); + GGML_API void ggml_reset(struct ggml_context * ctx); + GGML_API void ggml_free (struct ggml_context * ctx); + + GGML_API size_t ggml_used_mem(const struct ggml_context * ctx); + + GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx); + GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc); + + GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx); + GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx); + GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx); + + GGML_API struct ggml_tensor * ggml_new_tensor( + struct ggml_context * ctx, + enum ggml_type type, + int n_dims, + const int64_t *ne); + + GGML_API struct ggml_tensor * ggml_new_tensor_1d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_new_tensor_2d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_new_tensor_3d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_new_tensor_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes); + + GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src); + GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src); + + // Context tensor enumeration and lookup + GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx); + GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name); + + // Converts a flat index into coordinates + GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3); + + GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor); + GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor); + + GGML_API void * ggml_get_data (const struct ggml_tensor * tensor); + GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor); + + GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name); + GGML_ATTRIBUTE_FORMAT(2, 3) + GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...); + + // Tensor flags + GGML_API void ggml_set_input(struct ggml_tensor * tensor); + GGML_API void ggml_set_output(struct ggml_tensor * tensor); + GGML_API void ggml_set_param(struct ggml_tensor * tensor); + GGML_API void ggml_set_loss(struct ggml_tensor * tensor); + + // + // operations on tensors with backpropagation + // + + GGML_API struct ggml_tensor * ggml_dup( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_dup_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_add( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_type type); + + // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]] + GGML_API struct ggml_tensor * ggml_add_id( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add_inplace instead"); + + // dst = a + // view(dst, nb1, nb2, nb3, offset) += b + // return dst + GGML_API struct ggml_tensor * ggml_acc( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_acc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_sub( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sub_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sqr( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqr_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // return scalar + GGML_API struct ggml_tensor * ggml_sum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d] + GGML_API struct ggml_tensor * ggml_sum_rows( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cumsum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // mean along rows + GGML_API struct ggml_tensor * ggml_mean( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // argmax along rows + GGML_API struct ggml_tensor * ggml_argmax( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // count number of equal elements in a and b + GGML_API struct ggml_tensor * ggml_count_equal( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // if a is the same shape as b, and a is not parameter, return a + // otherwise, return a new tensor: repeat(a) to fit in b + GGML_API struct ggml_tensor * ggml_repeat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // repeat a to the specified shape + GGML_API struct ggml_tensor * ggml_repeat_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // sums repetitions in a into shape of b + GGML_API struct ggml_tensor * ggml_repeat_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride + + // concat a and b along dim + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_concat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int dim); + + GGML_API struct ggml_tensor * ggml_abs( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_abs_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_relu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_leaky_relu( + struct ggml_context * ctx, + struct ggml_tensor * a, float negative_slope, bool inplace); + + GGML_API struct ggml_tensor * ggml_relu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // GELU using erf (error function) when possible + // some backends may fallback to approximation based on Abramowitz and Stegun formula + GGML_API struct ggml_tensor * ggml_gelu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_erf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a - dy + // b - x + GGML_API struct ggml_tensor * ggml_silu_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // hardswish(x) = x * relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardswish( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // hardsigmoid(x) = relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardsigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + /** + * Truncates the fractional part of each element in the tensor (towards zero). + * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0 + * Similar to std::trunc in C/C++. + */ + + GGML_API struct ggml_tensor * ggml_trunc( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_trunc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + + + // xIELU activation function + // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0) + // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions + // that constrain the positive and negative source alpha values respectively + GGML_API struct ggml_tensor * ggml_xielu( + struct ggml_context * ctx, + struct ggml_tensor * a, + float alpha_n, + float alpha_p, + float beta, + float eps); + + // gated linear unit ops + // A: n columns, r rows, + // result is n / 2 columns, r rows, + // expects gate in second half of row, unless swapped is true + GGML_API struct ggml_tensor * ggml_glu( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_glu_op op, + bool swapped); + + GGML_API struct ggml_tensor * ggml_reglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_reglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // A: n columns, r rows, + // B: n columns, r rows, + GGML_API struct ggml_tensor * ggml_glu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_glu_op op); + + GGML_API struct ggml_tensor * ggml_reglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_erf_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_quick_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_oai( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float alpha, + float limit); + + // normalize along rows + GGML_API struct ggml_tensor * ggml_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // group normalize along ne0*ne1*n_groups + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_group_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + GGML_API struct ggml_tensor * ggml_group_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + // l2 normalize along rows + // used in rwkv v7 + GGML_API struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // a - x + // b - dy + GGML_API struct ggml_tensor * ggml_rms_norm_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float eps); + + // A: k columns, n rows => [ne03, ne02, n, k] + // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] + // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] + GGML_API struct ggml_tensor * ggml_mul_mat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // change the precision of a matrix multiplication + // set to GGML_PREC_F32 for higher precision (useful for phi-2) + GGML_API void ggml_mul_mat_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); + + // indirect matrix multiplication + GGML_API struct ggml_tensor * ggml_mul_mat_id( + struct ggml_context * ctx, + struct ggml_tensor * as, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + // A: m columns, n rows, + // B: p columns, n rows, + // result is m columns, p rows + GGML_API struct ggml_tensor * ggml_out_prod( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // + // operations on tensors without backpropagation + // + + GGML_API struct ggml_tensor * ggml_scale( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_scale_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // x = s * a + b + GGML_API struct ggml_tensor * ggml_scale_bias( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + GGML_API struct ggml_tensor * ggml_scale_bias_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_2d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // a -> b, return view(b) + GGML_API struct ggml_tensor * ggml_cpy( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // note: casting from f32 to i32 will discard the fractional part + GGML_API struct ggml_tensor * ggml_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_type type); + + // make contiguous + GGML_API struct ggml_tensor * ggml_cont( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // make contiguous, with new shape + GGML_API struct ggml_tensor * ggml_cont_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_cont_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_cont_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_cont_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // return view(a), b specifies the new shape + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_reshape_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_reshape_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // offset in bytes + GGML_API struct ggml_tensor * ggml_view_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + size_t nb1, // row stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_permute( + struct ggml_context * ctx, + struct ggml_tensor * a, + int axis0, + int axis1, + int axis2, + int axis3); + + // alias for ggml_permute(ctx, a, 1, 0, 2, 3) + GGML_API struct ggml_tensor * ggml_transpose( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // supports 4D a: + // a [n_embd, ne1, ne2, ne3] + // b I32 [n_rows, ne2, ne3, 1] + // + // return [n_embd, n_rows, ne2, ne3] + GGML_API struct ggml_tensor * ggml_get_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // data + struct ggml_tensor * b); // row indices + + GGML_API struct ggml_tensor * ggml_get_rows_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_get_rows result + struct ggml_tensor * b, // row indices + struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape + + // a TD [n_embd, ne1, ne2, ne3] + // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3 + // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1) + // + // undefined behavior if destination rows overlap + // + // broadcast: + // ne2 % ne11 == 0 + // ne3 % ne12 == 0 + // + // return view(a) + GGML_API struct ggml_tensor * ggml_set_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // destination + struct ggml_tensor * b, // source + struct ggml_tensor * c); // row indices + + GGML_API struct ggml_tensor * ggml_diag( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // set elements above the diagonal to -INF + GGML_API struct ggml_tensor * ggml_diag_mask_inf( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // set elements above the diagonal to 0 + GGML_API struct ggml_tensor * ggml_diag_mask_zero( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + GGML_API struct ggml_tensor * ggml_soft_max( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a [ne0, ne01, ne02, ne03] + // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional + // + // broadcast: + // ne02 % ne12 == 0 + // ne03 % ne13 == 0 + // + // fused soft_max(a*scale + mask*(ALiBi slope)) + // max_bias = 0.0f for no ALiBi + GGML_API struct ggml_tensor * ggml_soft_max_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API void ggml_soft_max_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // rotary position embedding + // if (mode & 1) - skip n_past elements (NOT SUPPORTED) + // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style + // + // b is an int32 vector with size a->ne[2], it contains the positions + GGML_API struct ggml_tensor * ggml_rope( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // RoPE operations with extended options + // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token] + // b is an int32 vector with size n_token + // c is freq factors (e.g. phi3-128k), (optional) + // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi + // + // pseudo-code for computing theta: + // for i in [0, n_dims/2): + // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims); + // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c + // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here + // + // other params are used by YaRN RoPE scaling, these default values will disable YaRN: + // freq_scale = 1.0f + // ext_factor = 0.0f + // attn_factor = 1.0f + // beta_fast = 0.0f + // beta_slow = 0.0f + // + // example: + // (marking: c = cos, s = sin, 0 = unrotated) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000] + // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs] + // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000] + // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss] + GGML_API struct ggml_tensor * ggml_rope_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // multi-dimensional RoPE, for Qwen-VL and similar vision models + // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX + // sections specify how many dimensions to rotate in each section: + // section length is equivalent to number of cos/sin pairs, NOT the number of dims + // (i.e. sum of 4 sections are expected to be n_dims/2) + // last sections can be 0, means ignored + // all other options are identical to ggml_rope_ext + // + // important note: + // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION + // if you need normal ordering, there are 2 methods: + // (1) split the tensor manually using ggml_view + // (2) permute the weight upon conversion + // - for VISION, n_dims must be head_size/2 + // + // example M-RoPE: + // given sections = [t=4, y=2, x=2, 0] + // given a single head with size = 18 --> [000000000000000000] + // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering) + // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering) + // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section + // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section + // + // example vision RoPE: + // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx] + // other values of n_dims are untested and is undefined behavior + // note: unlike MROPE, the theta for each dim is computed differently for each section + // in other words, idx used for theta: [0123] for y section, then [0123] for x section + GGML_API struct ggml_tensor * ggml_rope_multi( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext_inplace instead"); + + // compute correction dims for YaRN RoPE scaling + GGML_API void ggml_rope_yarn_corr_dims( + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]); + + // rotary position embedding backward, i.e compute dx from dy + // a - dy + GGML_API struct ggml_tensor * ggml_rope_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_rope result + struct ggml_tensor * b, // positions + struct ggml_tensor * c, // freq factors + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[4], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + + // clamp + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_clamp( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + + // im2col + // converts data into a format that effectively results in a convolution when combined with matrix multiplication + GGML_API struct ggml_tensor * ggml_im2col( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D, + enum ggml_type dst_type); + + GGML_API struct ggml_tensor * ggml_im2col_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // gradient of im2col output + int64_t * ne, // shape of im2col input + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D); + + // col2im_1d: scatter-add GEMM columns back to 1D signal + // a: [K*OC, T_in] (columns from matmul, K = a->ne[0]/OC) + // result: [T_out, OC] where T_out = (T_in - 1)*s0 + K - 2*p0 + GGML_API struct ggml_tensor * ggml_col2im_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // columns [K*OC, T_in] + int s0, // stride + int oc, // output channels + int p0); // padding to crop from both sides + + GGML_API struct ggml_tensor * ggml_conv_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + // conv_1d with padding = half + // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d) + GGML_API struct ggml_tensor* ggml_conv_1d_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s, // stride + int d); // dilation + + // depthwise + // TODO: this is very likely wrong for some cases! - needs more testing + GGML_API struct ggml_tensor * ggml_conv_1d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_transpose_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_im2col_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2, // dilation depth + enum ggml_type dst_type); + + // a: [OC*IC, KD, KH, KW] + // b: [N*IC, ID, IH, IW] + // result: [N*OC, OD, OH, OW] + GGML_API struct ggml_tensor * ggml_conv_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2 // dilation depth + ); + + // kernel size is a->ne[0] x a->ne[1] + // stride is equal to kernel size + // padding is zero + // example: + // a: 16 16 3 768 + // b: 1024 1024 3 1 + // res: 64 64 768 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // kernel size is a->ne[0] x a->ne[1] + // stride is 1 + // padding is half + // example: + // a: 3 3 256 256 + // b: 64 64 256 1 + // res: 64 64 256 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // depthwise (via im2col and mul_mat) + GGML_API struct ggml_tensor * ggml_conv_2d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + // Depthwise 2D convolution + // may be faster than ggml_conv_2d_dw, but not available in all backends + // a: KW KH 1 C convolution kernel + // b: W H C N input data + // res: W_out H_out C N + GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride0, + int stride1, + int pad0, + int pad1, + int dilation0, + int dilation1); + + GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride); + + GGML_API struct ggml_tensor * ggml_conv_2d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC] + struct ggml_tensor * b, // input data [W, H, C, N] + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_conv_3d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC] + struct ggml_tensor * b, // input [W, H, D, C * N] + int s0, // stride + int s1, + int s2, + int p0, // padding + int p1, + int p2, + int d0, // dilation + int d1, + int d2, + int n_channels, + int n_batch, + int n_channels_out); + + enum ggml_op_pool { + GGML_OP_POOL_MAX, + GGML_OP_POOL_AVG, + GGML_OP_POOL_COUNT, + }; + + GGML_API struct ggml_tensor * ggml_pool_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, // kernel size + int s0, // stride + int p0); // padding + + // the result will have 2*p0 padding for the first dimension + // and 2*p1 padding for the second dimension + GGML_API struct ggml_tensor * ggml_pool_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + GGML_API struct ggml_tensor * ggml_pool_2d_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * af, // "a"/input used in forward pass + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + enum ggml_scale_mode { + GGML_SCALE_MODE_NEAREST = 0, + GGML_SCALE_MODE_BILINEAR = 1, + GGML_SCALE_MODE_BICUBIC = 2, + + GGML_SCALE_MODE_COUNT + }; + + enum ggml_scale_flag { + GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8), + GGML_SCALE_FLAG_ANTIALIAS = (1 << 9), + }; + + // interpolate + // multiplies ne0 and ne1 by scale factor + GGML_API struct ggml_tensor * ggml_upscale( + struct ggml_context * ctx, + struct ggml_tensor * a, + int scale_factor, + enum ggml_scale_mode mode); + + // interpolate + // interpolate scale to specified dimensions + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int ne0, + int ne1, + int ne2, + int ne3, + enum ggml_scale_mode mode), + "use ggml_interpolate instead"); + + // Up- or downsamples the input to the specified size. + // 2D scale modes (eg. bilinear) are applied to the first two dimensions. + GGML_API struct ggml_tensor * ggml_interpolate( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...] + + // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0] + GGML_API struct ggml_tensor * ggml_pad( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + GGML_API struct ggml_tensor * ggml_pad_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3 + ); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_ext_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3); + + // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c] + GGML_API struct ggml_tensor * ggml_pad_reflect_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1); + + // Move tensor elements by an offset given for each dimension. Elements that + // are shifted beyond the last position are wrapped around to the beginning. + GGML_API struct ggml_tensor * ggml_roll( + struct ggml_context * ctx, + struct ggml_tensor * a, + int shift0, + int shift1, + int shift2, + int shift3); + + // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing + // zeroes everywhere outside the masked area + GGML_API struct ggml_tensor * ggml_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_tri_type type); + + // Fill tensor a with constant c + GGML_API struct ggml_tensor * ggml_fill( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + GGML_API struct ggml_tensor * ggml_fill_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151 + // timesteps: [N,] + // return: [N, dim] + GGML_API struct ggml_tensor * ggml_timestep_embedding( + struct ggml_context * ctx, + struct ggml_tensor * timesteps, + int dim, + int max_period); + + // sort rows + enum ggml_sort_order { + GGML_SORT_ORDER_ASC, + GGML_SORT_ORDER_DESC, + }; + + GGML_API struct ggml_tensor * ggml_argsort( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_sort_order order); + + // similar to ggml_top_k but implemented as `argsort` + `view` + GGML_API struct ggml_tensor * ggml_argsort_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + // top k elements per row + // note: the resulting top k indices are in no particular order + GGML_API struct ggml_tensor * ggml_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + GGML_API struct ggml_tensor * ggml_arange( + struct ggml_context * ctx, + float start, + float stop, + float step); + + // q: [n_embd_k, n_batch, n_head, ne3 ] + // k: [n_embd_k, n_kv, n_head_kv, ne3 ] + // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !! + // mask: [n_kv, n_batch, ne32, ne33] + // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !! + // + // broadcast: + // n_head % n_head_kv == 0 + // n_head % ne32 == 0 + // ne3 % ne33 == 0 + // + GGML_API struct ggml_tensor * ggml_flash_attn_ext( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + float scale, + float max_bias, + float logit_softcap); + + GGML_API void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( + const struct ggml_tensor * a); + + GGML_API void ggml_flash_attn_ext_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + // TODO: needs to be adapted to ggml_flash_attn_ext + GGML_API struct ggml_tensor * ggml_flash_attn_back( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * d, + bool masked); + + GGML_API struct ggml_tensor * ggml_ssm_conv( + struct ggml_context * ctx, + struct ggml_tensor * sx, + struct ggml_tensor * c); + + GGML_API struct ggml_tensor * ggml_ssm_scan( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * dt, + struct ggml_tensor * A, + struct ggml_tensor * B, + struct ggml_tensor * C, + struct ggml_tensor * ids); + + // partition into non-overlapping windows with padding if needed + // example: + // a: 768 64 64 1 + // w: 14 + // res: 768 14 14 25 + // used in sam + GGML_API struct ggml_tensor * ggml_win_part( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w); + + // reverse of ggml_win_part + // used in sam + GGML_API struct ggml_tensor * ggml_win_unpart( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w0, + int h0, + int w); + + GGML_API struct ggml_tensor * ggml_unary( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + GGML_API struct ggml_tensor * ggml_unary_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + // used in sam + GGML_API struct ggml_tensor * ggml_get_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + int qh, + int kh); + + // used in sam + GGML_API struct ggml_tensor * ggml_add_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv6( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * r, + struct ggml_tensor * tf, + struct ggml_tensor * td, + struct ggml_tensor * state); + + GGML_API struct ggml_tensor * ggml_gated_linear_attn( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * q, + struct ggml_tensor * g, + struct ggml_tensor * state, + float scale); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state); + + /* Solves a specific equation of the form Ax=B, where A is a triangular matrix + * without zeroes on the diagonal (i.e. invertible). + * B can have any number of columns, but must have the same number of rows as A + * If A is [n, n] and B is [n, m], then the result will be [n, m] as well + * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases + * where n > 100 sparingly, pre-chunk if necessary. + * + * If left = false, solves xA=B instead + * If lower = false, assumes upper triangular instead + * If uni = true, assumes diagonal of A to be all ones (will override actual values) + * + * TODO: currently only lower, right, non-unitriangular variant is implemented + */ + GGML_API struct ggml_tensor * ggml_solve_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + bool left, + bool lower, + bool uni); + + // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST] + // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306 + // + // state is a 3D tensor of shape (S_v*S_v*H, K, n_seqs): + // K == 1: output carries the final state only. + // K > 1: output carries K snapshot slots; the kernel writes the last min(n_tokens, K) + // per-token snapshots into the trailing slots + GGML_API struct ggml_tensor * ggml_gated_delta_net( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * g, + struct ggml_tensor * beta, + struct ggml_tensor * state); + + // custom operators + + typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata); + typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata); + typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata); + +#define GGML_N_TASKS_MAX (-1) + // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks + + GGML_API struct ggml_tensor * ggml_map_custom1( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + // loss function + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b); // labels + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b, // labels + struct ggml_tensor * c); // gradients of cross_entropy_loss result + + // AdamW optimizer step + // Paper: https://arxiv.org/pdf/1711.05101v3.pdf + // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html + GGML_API struct ggml_tensor * ggml_opt_step_adamw( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * m, + struct ggml_tensor * v, + struct ggml_tensor * adamw_params); // parameters such as the learning rate + + // stochastic gradient descent step (with weight decay) + GGML_API struct ggml_tensor * ggml_opt_step_sgd( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * sgd_params); // alpha, weight decay + + // build forward multiple tensors and select one of them for computing + // this is useful for creating graphs that have constant topology but compute different things based on the input + // ref: https://github.com/ggml-org/llama.cpp/pull/18550 + // + // nodes: + // | - build forward into the graph but do not compute + // c - build forward into the graph and compute + // + // | | ... c ... | + // | | ... c ... | + // | | ... c ... | + // [0 1 ... idx ... n-1] <-- ggml_build_forward_select(..., n, idx) + // c + // c + // + // example: + // struct ggml_tensor * curs[3]; + // + // curs[0] = compute0(...); + // curs[1] = compute1(...); + // curs[2] = compute2(...); + // + // int idx = select_branch(some_input); + // + // struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx); + // + GGML_API struct ggml_tensor * ggml_build_forward_select( + struct ggml_cgraph * cgraph, + struct ggml_tensor ** tensors, + int n_tensors, + int idx); + + GGML_API void ggml_build_forward_expand( + struct ggml_cgraph * cgraph, + struct ggml_tensor * tensor); + + GGML_API void ggml_build_backward_expand( + struct ggml_context * ctx, // context for gradient computation + struct ggml_cgraph * cgraph, + struct ggml_tensor ** grad_accs); + + // graph allocation in a context + GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false + GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads); + GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads); + GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst); + GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1 + GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph); + + GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph); + GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i] + GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph); + GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph); + + GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor); + + GGML_API size_t ggml_graph_overhead(void); + GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads); + + GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name); + GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + + // print info and performance information for the graph + GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph); + + // dump the graph into a file using the dot format + GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename); + + // TODO these functions were sandwiched in the old optimization interface, is there a better place for them? + typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data); + GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data); + + GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor); + + // + // quantization + // + + // - ggml_quantize_init can be called multiple times with the same type + // it will only initialize the quantization tables for the first call or after ggml_quantize_free + // automatically called by ggml_quantize_chunk for convenience + // + // - ggml_quantize_free will free any memory allocated by ggml_quantize_init + // call this at the end of the program to avoid memory leaks + // + // note: these are thread-safe + // + GGML_API void ggml_quantize_init(enum ggml_type type); + GGML_API void ggml_quantize_free(void); + + // some quantization type cannot be used without an importance matrix + GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type); + + // calls ggml_quantize_init internally (i.e. can allocate memory) + GGML_API size_t ggml_quantize_chunk( + enum ggml_type type, + const float * src, + void * dst, + int64_t start, + int64_t nrows, + int64_t n_per_row, + const float * imatrix); + +#ifdef __cplusplus + // restrict not standard in C++ +# if defined(__GNUC__) +# define GGML_RESTRICT __restrict__ +# elif defined(__clang__) +# define GGML_RESTRICT __restrict +# elif defined(_MSC_VER) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT +# endif +#else +# if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT restrict +# endif +#endif + typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + + struct ggml_type_traits { + const char * type_name; + int64_t blck_size; + int64_t blck_size_interleave; // interleave elements in blocks + size_t type_size; + bool is_quantized; + ggml_to_float_t to_float; + ggml_from_float_t from_float_ref; + }; + + GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type); + + // ggml threadpool + // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend + // the goal should be to create an API that other backends can use move everything to the ggml base + + // scheduling priorities + enum ggml_sched_priority { + GGML_SCHED_PRIO_LOW = -1, + GGML_SCHED_PRIO_NORMAL, + GGML_SCHED_PRIO_MEDIUM, + GGML_SCHED_PRIO_HIGH, + GGML_SCHED_PRIO_REALTIME + }; + + // threadpool params + // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults + struct ggml_threadpool_params { + bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings) + int n_threads; // number of threads + enum ggml_sched_priority prio; // thread priority + uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling) + bool strict_cpu; // strict cpu placement + bool paused; // start in paused state + }; + + struct ggml_threadpool; // forward declaration, see ggml.c + + typedef struct ggml_threadpool * ggml_threadpool_t; + + GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads); + GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads); + GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/gguf.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/gguf.h new file mode 100644 index 00000000..67851ba6 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/gguf.h @@ -0,0 +1,210 @@ +// This file contains functionality related to "GGUF" files, the binary file format used by ggml. +// GGUF files have the following structure: +// +// 1. File magic "GGUF" (4 bytes). +// 2. File version (uint32_t). +// 3. Number of ggml tensors in file (int64_t). +// 4. Number of key-value-pairs in file (int64_t). +// 5. For each KV pair: +// 1. The key (string). +// 2. The value type (gguf_type). +// 3a. If the value type is GGUF_TYPE_ARRAY: +// 1. The type of the array (gguf_type). +// 2. The number of elements in the array (uint64_t). +// 3. The binary representation of each element in the array. +// 3b. Otherwise: +// 1. The binary representation of the value. +// 6. For each ggml tensor: +// 1. The tensor name (string). +// 2. The number of dimensions of the tensor (uint32_t). +// 3. For each dimension: +// 1. The size of the tensor in the dimension (int64_t). +// 4. The tensor data type (ggml_type). +// 5. The tensor data offset in the tensor data binary blob (uint64_t). +// 7. The tensor data binary blob (optional, aligned). +// +// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator. +// All enums are stored as int32_t. +// All bool values are stored as int8_t. +// If the special key "general.alignment" (uint32_t) is defined it is used for alignment, +// otherwise GGUF_DEFAULT_ALIGNMENT is used. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" + +#include +#include + +#define GGUF_MAGIC "GGUF" +#define GGUF_VERSION 3 + +#define GGUF_KEY_GENERAL_ALIGNMENT "general.alignment" + +#define GGUF_DEFAULT_ALIGNMENT 32 + +#ifdef __cplusplus +extern "C" { +#endif + + // types that can be stored as GGUF KV data + enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, + GGUF_TYPE_UINT64 = 10, + GGUF_TYPE_INT64 = 11, + GGUF_TYPE_FLOAT64 = 12, + GGUF_TYPE_COUNT, // marks the end of the enum + }; + + struct gguf_context; + + struct gguf_init_params { + bool no_alloc; + + // if not NULL, create a ggml_context and allocate the tensor data in it + struct ggml_context ** ctx; + }; + + // callback to simulate or wrap a FILE pointer - read up to `len` bytes at `offset` into `output` and return the number of bytes read + typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len); + + GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); + + // max_chunk_read is the maximum number of bytes that the GGUF code will read at once from the callback, a value of 0 means no limit + GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params); + + GGML_API void gguf_free(struct gguf_context * ctx); + + GGML_API const char * gguf_type_name(enum gguf_type type); + + GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx); + GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx); + GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); // padded to gguf_get_alignment if and only if the gguf_context contains at least one tensor + + GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx); + GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found + GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id); + + GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id); + GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id); + + // will abort if the wrong type is used for the key + GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id); + GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id); + GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id); + GGML_API size_t gguf_get_arr_n (const struct gguf_context * ctx, int64_t key_id); + + // get raw pointer to the first element of the array with the given key_id + // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference) + GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id); + + // get ith C string from array with given key_id + GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i); + + GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx); + GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found + GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id); + GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id); + + // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist) + GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key); + + // overrides an existing KV pair or adds a new one, the new KV pair is always at the back + GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val); + GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val); + GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val); + GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val); + GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val); + GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val); + GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val); + GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val); + GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val); + GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val); + GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val); + GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val); + + // creates a new array with n elements of the given type and copies the corresponding number of bytes from data + GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n); + + // creates a new array with n strings and copies the corresponding strings from data + GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n); + + // set or add KV pairs from another context + GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src); + + // add tensor to GGUF context, tensor name must be unique + GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor); + + // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated + // in such a way that the tensor data remains as one contiguous block (except for padding) + GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type); + + // assumes that at least gguf_get_tensor_size bytes can be read from data + GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data); + + // writing gguf files can be done in 3 ways: + // + // - write the entire gguf_context to a binary file in a single pass: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ false); + // + // - write only the meta data to a file, then re-open the file and append the tensor data: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ true); + // FILE * f = fopen(fname, "ab"); + // fwrite(f, ...); // write tensor data + // fclose(f); + // + // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data: + // + // FILE * f = fopen(fname, "wb"); + // const size_t size_meta = gguf_get_meta_size(ctx); + // fseek(f, size_meta, SEEK_SET); + // fwrite(f, ...); // write tensor data + // void * data = malloc(size_meta); + // gguf_get_meta_data(ctx, data); + // rewind(f); + // fwrite(data, 1, data, f); + // free(data); + // fclose(f); + // + + // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); + GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); + + // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding + GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx); + + // writes the meta data to pointer "data" + GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/llama.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/llama.h new file mode 100644 index 00000000..27e48067 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/llama.h @@ -0,0 +1,1588 @@ +#ifndef LLAMA_H +#define LLAMA_H + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-backend.h" +#include "ggml-opt.h" +#include "gguf.h" + +#include +#include +#include +#include + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define LLAMA_API __declspec(dllexport) +# else +# define LLAMA_API __declspec(dllimport) +# endif +# else +# define LLAMA_API __attribute__ ((visibility ("default"))) +# endif +#else +# define LLAMA_API +#endif + +#ifdef __GNUC__ +# define DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define DEPRECATED(func, hint) func +#endif + +#define LLAMA_DEFAULT_SEED 0xFFFFFFFF + +#define LLAMA_TOKEN_NULL -1 + +#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' +#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' +#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq' + +#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN +#define LLAMA_SESSION_VERSION 9 + +#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ +#define LLAMA_STATE_SEQ_VERSION 2 + +#ifdef __cplusplus +extern "C" { +#endif + + // + // C interface + // + // TODO: show sample usage + // + + struct llama_vocab; + struct llama_model; + struct llama_context; + struct llama_sampler; + + typedef struct llama_memory_i * llama_memory_t; + + typedef int32_t llama_pos; + typedef int32_t llama_token; + typedef int32_t llama_seq_id; + + enum llama_vocab_type { + LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab + LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback + LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE + LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece + LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram + LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization + LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming + }; + + enum llama_rope_type { + LLAMA_ROPE_TYPE_NONE = -1, + LLAMA_ROPE_TYPE_NORM = 0, + LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, + LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE, + LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE, + LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION, + }; + + enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file + LLAMA_TOKEN_TYPE_UNDEFINED = 0, + LLAMA_TOKEN_TYPE_NORMAL = 1, + LLAMA_TOKEN_TYPE_UNKNOWN = 2, + LLAMA_TOKEN_TYPE_CONTROL = 3, + LLAMA_TOKEN_TYPE_USER_DEFINED = 4, + LLAMA_TOKEN_TYPE_UNUSED = 5, + LLAMA_TOKEN_TYPE_BYTE = 6, + }; + + enum llama_token_attr { + LLAMA_TOKEN_ATTR_UNDEFINED = 0, + LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0, + LLAMA_TOKEN_ATTR_UNUSED = 1 << 1, + LLAMA_TOKEN_ATTR_NORMAL = 1 << 2, + LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL? + LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4, + LLAMA_TOKEN_ATTR_BYTE = 1 << 5, + LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6, + LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7, + LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8, + LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9, + }; + + // model file types + enum llama_ftype { + LLAMA_FTYPE_ALL_F32 = 0, + LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed + // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed + LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors + LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors + //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack + //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack + //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack + LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors + LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors + LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors + LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors + + LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file + }; + + enum llama_rope_scaling_type { + LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, + LLAMA_ROPE_SCALING_TYPE_NONE = 0, + LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, + LLAMA_ROPE_SCALING_TYPE_YARN = 2, + LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, + LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE, + }; + + enum llama_pooling_type { + LLAMA_POOLING_TYPE_UNSPECIFIED = -1, + LLAMA_POOLING_TYPE_NONE = 0, + LLAMA_POOLING_TYPE_MEAN = 1, + LLAMA_POOLING_TYPE_CLS = 2, + LLAMA_POOLING_TYPE_LAST = 3, + LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph + }; + + enum llama_attention_type { + LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1, + LLAMA_ATTENTION_TYPE_CAUSAL = 0, + LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1, + }; + + enum llama_flash_attn_type { + LLAMA_FLASH_ATTN_TYPE_AUTO = -1, + LLAMA_FLASH_ATTN_TYPE_DISABLED = 0, + LLAMA_FLASH_ATTN_TYPE_ENABLED = 1, + }; + + LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); + + enum llama_split_mode { + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_TENSOR = 3, + }; + + enum llama_context_type { + LLAMA_CONTEXT_TYPE_DEFAULT = 0, + LLAMA_CONTEXT_TYPE_MTP = 1, + }; + + // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) + typedef struct llama_token_data { + llama_token id; // token id + float logit; // log-odds of the token + float p; // probability of the token + } llama_token_data; + + typedef struct llama_token_data_array { + // TODO: consider SoA + // NOTE: this pointer can be modified by the samplers + llama_token_data * data; + size_t size; + int64_t selected; // this is the index in the data array (i.e. not the token id) + bool sorted; // note: do not assume the data is sorted - always check this flag + } llama_token_data_array; + + typedef bool (*llama_progress_callback)(float progress, void * user_data); + + // Input data for llama_encode/llama_decode + // A llama_batch object can contain input about one or many sequences + // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens + // + // - token : the token ids of the input (used when embd is NULL) + // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) + // - pos : the positions of the respective token in the sequence + // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode) + // - seq_id : the sequence to which the respective token belongs + // (if set to NULL, the sequence ID will be assumed to be 0) + // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output + // (if set to NULL: + // - if embeddings: all tokens are output + // - if not: only the last token is output + // ) + // + typedef struct llama_batch { + int32_t n_tokens; + + llama_token * token; + float * embd; + llama_pos * pos; + int32_t * n_seq_id; + llama_seq_id ** seq_id; + int8_t * logits; // TODO: rename this to "output" + } llama_batch; + + enum llama_model_kv_override_type { + LLAMA_KV_OVERRIDE_TYPE_INT, + LLAMA_KV_OVERRIDE_TYPE_FLOAT, + LLAMA_KV_OVERRIDE_TYPE_BOOL, + LLAMA_KV_OVERRIDE_TYPE_STR, + }; + + enum llama_model_meta_key { + LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE, + LLAMA_MODEL_META_KEY_SAMPLING_TOP_K, + LLAMA_MODEL_META_KEY_SAMPLING_TOP_P, + LLAMA_MODEL_META_KEY_SAMPLING_MIN_P, + LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY, + LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD, + LLAMA_MODEL_META_KEY_SAMPLING_TEMP, + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N, + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA, + }; + + struct llama_model_kv_override { + enum llama_model_kv_override_type tag; + + char key[128]; + + union { + int64_t val_i64; + double val_f64; + bool val_bool; + char val_str[128]; + }; + }; + + struct llama_model_tensor_buft_override { + const char * pattern; + ggml_backend_buffer_type_t buft; + }; + + struct llama_model_params { + // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) + ggml_backend_dev_t * devices; + + // NULL-terminated list of buffer types to use for tensors that match a pattern + const struct llama_model_tensor_buft_override * tensor_buft_overrides; + + int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers + enum llama_split_mode split_mode; // how to split the model across multiple GPUs + + // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE + int32_t main_gpu; + + // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() + const float * tensor_split; + + // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. + // If the provided progress_callback returns true, model loading continues. + // If it returns false, model loading is immediately aborted. + llama_progress_callback progress_callback; + + // context pointer passed to the progress callback + void * progress_callback_user_data; + + // override key-value pairs of the model meta data + const struct llama_model_kv_override * kv_overrides; + + // Keep the booleans together to avoid misalignment during copy-by-value. + bool vocab_only; // only load the vocabulary, no weights + bool use_mmap; // use mmap if possible + bool use_direct_io; // use direct io, takes precedence over use_mmap when supported + bool use_mlock; // force system to keep model in RAM + bool check_tensors; // validate model tensor data + bool use_extra_bufts; // use extra buffer types (used for weight repacking) + bool no_host; // bypass host buffer allowing extra buffers to be used + bool no_alloc; // only load metadata and simulate memory allocations + }; + + struct llama_sampler_seq_config { + llama_seq_id seq_id; + struct llama_sampler * sampler; + }; + + // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations + // https://github.com/ggml-org/llama.cpp/pull/7544 + struct llama_context_params { + uint32_t n_ctx; // text context, 0 = from model + uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode + uint32_t n_ubatch; // physical maximum batch size + uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing + + enum llama_context_type ctx_type; // set the context type (e.g. MTP) + enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` + enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id + enum llama_attention_type attention_type; // attention type to use for embeddings + enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention + + // ref: https://github.com/ggml-org/llama.cpp/pull/2054 + float rope_freq_base; // RoPE base frequency, 0 = from model + float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model + float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model + float yarn_attn_factor; // YaRN magnitude scaling factor + float yarn_beta_fast; // YaRN low correction dim + float yarn_beta_slow; // YaRN high correction dim + uint32_t yarn_orig_ctx; // YaRN original context size + float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default) + + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; + + enum ggml_type type_k; // data type for K cache [EXPERIMENTAL] + enum ggml_type type_v; // data type for V cache [EXPERIMENTAL] + + // Abort callback + // if it returns true, execution of llama_decode() will be aborted + // currently works only with CPU execution + ggml_abort_callback abort_callback; + void * abort_callback_data; + + // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. + bool embeddings; // if true, extract embeddings (together with logits) + bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU + bool no_perf; // measure performance timings + bool op_offload; // offload host tensor operations to device + bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) + // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases + // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 + bool kv_unified; // use a unified buffer across the input sequences when computing the attention + // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix + // ref: https://github.com/ggml-org/llama.cpp/pull/14363 + + // [EXPERIMENTAL] + // backend sampler chain configuration (make sure the caller keeps the sampler chains alive) + // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init) + struct llama_sampler_seq_config * samplers; + size_t n_samplers; + + // a source/target/parent context + // can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts + struct llama_context * ctx_other; + }; + + struct llama_model_tensor_override { + const char * pattern; + enum ggml_type type; + }; + + struct llama_model_imatrix_data { + const char * name; + const float * data; + size_t size; + }; + + // model quantization parameters + typedef struct llama_model_quantize_params { + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // token embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + bool keep_split; // quantize to the same number of shards + bool dry_run; // calculate and show the final quantization size without performing quantization + const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data + const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides + const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides + const int32_t * prune_layers; // pointer to layer indices to prune + } llama_model_quantize_params; + + typedef struct llama_logit_bias { + llama_token token; + float bias; + } llama_logit_bias; + + typedef struct llama_sampler_chain_params { + bool no_perf; // whether to measure performance timings + } llama_sampler_chain_params; + + // used in chat template + typedef struct llama_chat_message { + const char * role; + const char * content; + } llama_chat_message; + + // lora adapter + struct llama_adapter_lora; + + // Helpers for getting default parameters + // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172) + LLAMA_API struct llama_model_params llama_model_default_params(void); + LLAMA_API struct llama_context_params llama_context_default_params(void); + LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void); + LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void); + + // Initialize the llama + ggml backend + // If numa is true, use NUMA optimizations + // Call once at the start of the program + LLAMA_API void llama_backend_init(void); + + // Call once at the end of the program - currently only used for MPI + LLAMA_API void llama_backend_free(void); + + //optional: + LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa); + + // Optional: an auto threadpool gets created in ggml if not passed explicitly + LLAMA_API void llama_attach_threadpool( + struct llama_context * ctx, + ggml_threadpool_t threadpool, + ggml_threadpool_t threadpool_batch); + + LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); + + typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata); + + // Create a new model from GGUF metadata as well as a function to set the tensor data + // - tensors are created as GGML_TYPE_F32 by default, + // override by adding a tensor with the same name but a different name to the context + LLAMA_API struct llama_model * llama_model_init_from_user( + struct gguf_context * metadata, + llama_model_set_tensor_data_t set_tensor_data, // function to initialize tensor data with + void * set_tensor_data_ud, // userdata for function + struct llama_model_params params); + + DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( + const char * path_model, + struct llama_model_params params), + "use llama_model_load_from_file instead"); + + // Load a model from a file + // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf + // If the split file name does not follow this pattern, use llama_model_load_from_splits + LLAMA_API struct llama_model * llama_model_load_from_file( + const char * path_model, + struct llama_model_params params); + + // Load a model from an open FILE pointer + LLAMA_API struct llama_model * llama_model_load_from_file_ptr( + FILE * file, + struct llama_model_params params); + + // Load a model from multiple splits (support custom naming scheme) + // The paths must be in the correct order + LLAMA_API struct llama_model * llama_model_load_from_splits( + const char ** paths, + size_t n_paths, + struct llama_model_params params); + + LLAMA_API void llama_model_save_to_file( + const struct llama_model * model, + const char * path_model); + + DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model), + "use llama_model_free instead"); + + LLAMA_API void llama_model_free(struct llama_model * model); + + LLAMA_API struct llama_context * llama_init_from_model( + struct llama_model * model, + struct llama_context_params params); + + DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model( + struct llama_model * model, + struct llama_context_params params), + "use llama_init_from_model instead"); + + // Frees all allocated memory + LLAMA_API void llama_free(struct llama_context * ctx); + + LLAMA_API int64_t llama_time_us(void); + + LLAMA_API size_t llama_max_devices(void); + LLAMA_API size_t llama_max_parallel_sequences(void); + LLAMA_API size_t llama_max_tensor_buft_overrides(void); + + LLAMA_API bool llama_supports_mmap (void); + LLAMA_API bool llama_supports_mlock (void); + LLAMA_API bool llama_supports_gpu_offload(void); + LLAMA_API bool llama_supports_rpc (void); + + // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions + // In some cases the requested values via llama_context_params may differ from the actual values used by the context + // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732 + LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx); + + DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); + DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); + DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); + DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); + + DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); + + LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); + LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx); + LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type + + LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); + LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model); + + LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model); + + // Get the model's RoPE frequency scaling factor + LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); + + // Returns the number of classifier outputs (only valid for classifier models) + // Undefined behavior for non-classifier models + LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); + + // Returns label of classifier output by index ( 1` + // p0 < 0 : [0, p1] + // p1 < 0 : [p0, inf) + LLAMA_API void llama_memory_seq_div( + llama_memory_t mem, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d); + + // Returns the smallest position present in the memory for the specified sequence + // This is typically non-zero only for SWA caches + // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory + // Return -1 if the sequence is empty + LLAMA_API llama_pos llama_memory_seq_pos_min( + llama_memory_t mem, + llama_seq_id seq_id); + + // Returns the largest position present in the memory for the specified sequence + // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory + // Return -1 if the sequence is empty + LLAMA_API llama_pos llama_memory_seq_pos_max( + llama_memory_t mem, + llama_seq_id seq_id); + + // Check if the memory supports shifting + LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); + + // + // State / sessions + // + + // Returns the *actual* size in bytes of the state + // (logits, embedding and memory) + // Only use when saving the state, not when restoring it, otherwise the size may be too small. + LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); + LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx), + "use llama_state_get_size instead"); + + // Copies the state to the specified destination address. + // Destination needs to have allocated enough memory. + // Returns the number of bytes copied + LLAMA_API size_t llama_state_get_data( + struct llama_context * ctx, + uint8_t * dst, + size_t size); + LLAMA_API DEPRECATED(size_t llama_copy_state_data( + struct llama_context * ctx, + uint8_t * dst), + "use llama_state_get_data instead"); + + // Set the state reading from the specified address + // Returns the number of bytes read + LLAMA_API size_t llama_state_set_data( + struct llama_context * ctx, + const uint8_t * src, + size_t size); + LLAMA_API DEPRECATED(size_t llama_set_state_data( + struct llama_context * ctx, + const uint8_t * src), + "use llama_state_set_data instead"); + + // Save/load session file + LLAMA_API bool llama_state_load_file( + struct llama_context * ctx, + const char * path_session, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + LLAMA_API DEPRECATED(bool llama_load_session_file( + struct llama_context * ctx, + const char * path_session, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out), + "use llama_state_load_file instead"); + + LLAMA_API bool llama_state_save_file( + struct llama_context * ctx, + const char * path_session, + const llama_token * tokens, + size_t n_token_count); + LLAMA_API DEPRECATED(bool llama_save_session_file( + struct llama_context * ctx, + const char * path_session, + const llama_token * tokens, + size_t n_token_count), + "use llama_state_save_file instead"); + + // Get the exact size needed to copy the state of a single sequence + LLAMA_API size_t llama_state_seq_get_size( + struct llama_context * ctx, + llama_seq_id seq_id); + + // Copy the state of a single sequence into the specified buffer + LLAMA_API size_t llama_state_seq_get_data( + struct llama_context * ctx, + uint8_t * dst, + size_t size, + llama_seq_id seq_id); + + // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence + // Returns: + // - Positive: Ok + // - Zero: Failed to load + LLAMA_API size_t llama_state_seq_set_data( + struct llama_context * ctx, + const uint8_t * src, + size_t size, + llama_seq_id dest_seq_id); + + LLAMA_API size_t llama_state_seq_save_file( + struct llama_context * ctx, + const char * filepath, + llama_seq_id seq_id, + const llama_token * tokens, + size_t n_token_count); + + LLAMA_API size_t llama_state_seq_load_file( + struct llama_context * ctx, + const char * filepath, + llama_seq_id dest_seq_id, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + +#define LLAMA_STATE_SEQ_FLAGS_NONE 0 + +// for backwards-compat +#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 + +// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) +#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 + +// Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load). +// Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag. +#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 + + typedef uint32_t llama_state_seq_flags; + + LLAMA_API size_t llama_state_seq_get_size_ext( + struct llama_context * ctx, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_get_data_ext( + struct llama_context * ctx, + uint8_t * dst, + size_t size, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_set_data_ext( + struct llama_context * ctx, + const uint8_t * src, + size_t size, + llama_seq_id dest_seq_id, + llama_state_seq_flags flags); + + // + // Decoding + // + + // Return batch for single sequence of tokens + // The sequence ID will be fixed to 0 + // The position of the tokens will be tracked automatically by llama_decode + // + // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it + // + LLAMA_API struct llama_batch llama_batch_get_one( + llama_token * tokens, + int32_t n_tokens); + + // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens + // Each token can be assigned up to n_seq_max sequence ids + // The batch has to be freed with llama_batch_free() + // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) + // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token + // The rest of the llama_batch members are allocated with size n_tokens + // All members are left uninitialized + LLAMA_API struct llama_batch llama_batch_init( + int32_t n_tokens, + int32_t embd, + int32_t n_seq_max); + + // Frees a batch of tokens allocated with llama_batch_init() + LLAMA_API void llama_batch_free(struct llama_batch batch); + + // Process a batch of tokens. + // In contrast to llama_decode() - this call does not use KV cache. + // For encode-decoder contexts, processes the batch using the encoder. + // Can store the encoder output internally for later use by the decoder's cross-attention layers. + // 0 - success + // < 0 - error. the memory state is restored to the state before this call + LLAMA_API int32_t llama_encode( + struct llama_context * ctx, + struct llama_batch batch); + + // Process a batch of tokens. + // Requires the context to have a memory. + // For encode-decoder contexts, processes the batch using the decoder. + // Positive return values does not mean a fatal error, but rather a warning. + // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context + // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + // Upon other return values, the memory state is restored to the state before this call + // 0 - success + // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) + // 2 - aborted (processed ubatches will remain in the context's memory) + // -1 - invalid input batch + // < -1 - fatal error (processed ubatches will remain in the context's memory) + LLAMA_API int32_t llama_decode( + struct llama_context * ctx, + struct llama_batch batch); + + // Set the number of threads used for decoding + // n_threads is the number of threads used for generation (single token) + // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) + LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch); + + // Get the number of threads used for generation of a single token. + LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); + + // Get the number of threads used for prompt and batch processing (multiple token). + LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); + + // Set whether the context outputs embeddings or not + // TODO: rename to avoid confusion with llama_get_embeddings() + LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); + + // Set whether to use causal attention or not + // If set to true, the model will only attend to the past tokens + LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); + + // Set whether the model is in warmup mode or not + // If true, all model tensors are activated during llama_decode() to load and cache their weights. + // + // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models, + // so it is generally not recommended to use in practice. will be removed in the future + DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup), + "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]"); + + // Set abort callback + LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); + + // Wait until all computations are finished + // This is automatically done when using one of the functions below to obtain the computation results + // and is not necessary to call it explicitly in most cases + LLAMA_API void llama_synchronize(struct llama_context * ctx); + + // Token logits obtained from the last call to llama_decode() + // The logits for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have appeared in the batch. + // Rows: number of tokens for which llama_batch.logits[i] != 0 + // Cols: n_vocab + // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) + LLAMA_API float * llama_get_logits(struct llama_context * ctx); + + // Logits for the ith token. For positive indices, Equivalent to: + // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab + // Negative indices can be used to access logits in reverse order, -1 is the last logit. + // returns NULL for invalid ids. + LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); + + // Get all output token embeddings. + // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, + // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have appeared in the batch. + // shape: [n_outputs*n_embd] + // Otherwise, returns NULL. + // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) + LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); + + // Get the embeddings for the ith token. For positive indices, Equivalent to: + // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd + // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding. + // shape: [n_embd] (1-dimensional) + // returns NULL for invalid ids. + LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); + + // Get the embeddings for a sequence id + // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE + // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence + // otherwise: float[n_embd] (1-dimensional) + LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); + + // + // backend sampling API [EXPERIMENTAL] + // note: use only if the llama_context was created with at least one llama_sampler_seq_config + // + + // Get the backend sampled token for the ith token. + // Returns LLAMA_TOKEN_NULL if no token was sampled. + LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled probabilities for the ith token + // The index matches llama_get_sampled_token_ith(). + // Returns NULL if no probabilities were generated. + LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled logits for the ith token + // Returns NULL if no logits were sampled. + LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled candidates (token ids) for the ith token + // These are needed to map probability/logit indices to vocab token ids. + // Returns NULL if no candidates were sampled. + LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i); + + // + // Vocab + // + + LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token); + + LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token); + + LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token); + + // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) + LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token); + + // Identify if Token Id is a control token or a render-able token + LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token); + + // Special tokens + LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence + LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence + LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn + LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator + LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line + LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding + LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask + + LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab); + LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab); + LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab); + + LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab); + + DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); + DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); + DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); + DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); + DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); + DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); + DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); + DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); + DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); + DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); + DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); + DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); + + // CLS is equivalent to BOS + DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification + "use llama_vocab_bos instead"); + + // + // Tokenization + // + // The API is thread-safe. + // + + /// @details Convert the provided text into tokens. + /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. + /// @return Returns the number of tokens on success, no more than n_tokens_max + /// @return Returns a negative number on failure - the number of tokens that would have been returned + /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit) + /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so. + /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated + /// as plaintext. Does not insert a leading space. + LLAMA_API int32_t llama_tokenize( + const struct llama_vocab * vocab, + const char * text, + int32_t text_len, + llama_token * tokens, + int32_t n_tokens_max, + bool add_special, + bool parse_special); + + // Token Id -> Piece. + // Uses the vocabulary in the provided context. + // Does not write null terminator to the buffer. + // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') + // @param special If true, special tokens are rendered in the output. + LLAMA_API int32_t llama_token_to_piece( + const struct llama_vocab * vocab, + llama_token token, + char * buf, + int32_t length, + int32_t lstrip, + bool special); + + /// @details Convert the provided tokens into text (inverse of llama_tokenize()). + /// @param text The char pointer must be large enough to hold the resulting text. + /// @return Returns the number of chars/bytes on success, no more than text_len_max. + /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned. + /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. + /// @param unparse_special If true, special tokens are rendered in the output. + LLAMA_API int32_t llama_detokenize( + const struct llama_vocab * vocab, + const llama_token * tokens, + int32_t n_tokens, + char * text, + int32_t text_len_max, + bool remove_special, + bool unparse_special); + + // + // Chat templates + // + + /// Apply chat template. Inspired by hf apply_chat_template() on python. + /// + /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template + /// @param tmpl A Jinja template to use for this chat. + /// @param chat Pointer to a list of multiple llama_chat_message + /// @param n_msg Number of llama_chat_message in this chat + /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. + /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages) + /// @param length The size of the allocated buffer + /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. + LLAMA_API int32_t llama_chat_apply_template( + const char * tmpl, + const struct llama_chat_message * chat, + size_t n_msg, + bool add_ass, + char * buf, + int32_t length); + + // Get list of built-in chat templates + LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len); + + // + // Sampling API + // + // Sample usage: + // + // // prepare the sampling chain at the start + // auto sparams = llama_sampler_chain_default_params(); + // + // llama_sampler * smpl = llama_sampler_chain_init(sparams); + // + // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50)); + // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); + // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8)); + // + // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat" + // // this sampler will be responsible to select the actual token + // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); + // + // ... + // + // // decoding loop: + // while (...) { + // ... + // + // llama_decode(ctx, batch); + // + // // sample from the logits of the last token in the batch + // const llama_token id = llama_sampler_sample(smpl, ctx, -1); + // + // ... + // } + // + // llama_sampler_free(smpl); + // + + typedef void * llama_sampler_context_t; + + struct llama_sampler_data { + struct ggml_tensor * logits; + struct ggml_tensor * probs; + struct ggml_tensor * sampled; + struct ggml_tensor * candidates; + }; + + // user code can implement the interface below in order to create custom llama_sampler + struct llama_sampler_i { + const char * (*name) (const struct llama_sampler * smpl); // can be NULL + void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL + void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required + void (*reset) ( struct llama_sampler * smpl); // can be NULL + struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL + void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL + + // [EXPERIMENTAL] + // backend sampling interface: + + // return true if the backend supports all ops needed by the sampler + // note: call once per sampler + bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft); + + // call after .backend_apply() + void (*backend_accept)( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct ggml_tensor * selected_token); + + // call after .backend_init() + void (*backend_apply)( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct llama_sampler_data * data); + + // called before graph execution to set inputs for the current ubatch + void (*backend_set_input)(struct llama_sampler * smpl); + }; + + struct llama_sampler { + struct llama_sampler_i * iface; + + llama_sampler_context_t ctx; + }; + + // [EXPERIMENTAL] + // attach a sampler to the context + // note: prefer initializing the context with llama_context_params.samplers when possible + LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl); + + // mirror of llama_sampler_i: + LLAMA_API struct llama_sampler * llama_sampler_init ( struct llama_sampler_i * iface, llama_sampler_context_t ctx); + LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); + LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); + LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); + LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); + LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); + // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) + LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); + + // llama_sampler_chain + // a type of llama_sampler that can chain multiple samplers one after another + + LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params); + + // important: takes ownership of the sampler object and will free it when llama_sampler_free is called + LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl); + + // return NULL if: + // - the sampler is NULL + // - the sampler is not a llama_sampler_chain + // - the index is out of bounds, unless i == -1 + // - if i == -1, returns the chain itself (can be used to check if the sampler is a chain) + LLAMA_API struct llama_sampler * llama_sampler_chain_get( struct llama_sampler * chain, int32_t i); + + // the total number of samplers in the chain + LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); + + // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed + LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i); + + // available samplers: + + LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void); + + /// seed == LLAMA_DEFAULT_SEED to use a random seed. + LLAMA_API struct llama_sampler * llama_sampler_init_dist(uint32_t seed); + + /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// Setting k <= 0 makes this a noop + LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); + + /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep); + + /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 + LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep); + + /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep); + + /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf + LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); + + /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. + LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent); + + /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 + LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed); + + /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 + LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n); + + /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API struct llama_sampler * llama_sampler_init_mirostat( + int32_t n_vocab, + uint32_t seed, + float tau, + float eta, + int32_t m); + + /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2( + uint32_t seed, + float tau, + float eta); + + /// @details Initializes a GBNF grammar, see grammars/README.md for details. + /// @param vocab The vocabulary that this grammar will be used with. + /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. + /// @param grammar_root The name of the start symbol for the grammar. + LLAMA_API struct llama_sampler * llama_sampler_init_grammar( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root); + + DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root, + const char ** trigger_words, + size_t num_trigger_words, + const llama_token * trigger_tokens, + size_t num_trigger_tokens), + "use llama_sampler_init_grammar_lazy_patterns instead"); + + + /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 + /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. + /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. + LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root, + const char ** trigger_patterns, + size_t num_trigger_patterns, + const llama_token * trigger_tokens, + size_t num_trigger_tokens); + + + /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. + LLAMA_API struct llama_sampler * llama_sampler_init_penalties( + int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) + float penalty_repeat, // 1.0 = disabled + float penalty_freq, // 0.0 = disabled + float penalty_present); // 0.0 = disabled + + /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 + LLAMA_API struct llama_sampler * llama_sampler_init_dry( + const struct llama_vocab * vocab, + int32_t n_ctx_train, + float dry_multiplier, + float dry_base, + int32_t dry_allowed_length, + int32_t dry_penalty_last_n, + const char ** seq_breakers, + size_t num_breakers); + + /// adaptive-p: select tokens near a configurable target probability over time. + /// + /// the adaptive-p sampler transforms the token probability distribution to favor tokens + /// that fall near a user-configurable probability target. + /// + /// internally, the sampler maintains an exponential moving average of the *ORIGINAL* + /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an + /// adapted target probability at each sampling step, thus maintaining the desired target + /// probability over time. + /// + /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last + /// in the sampler chain (like mirostat, dist, greedy). + /// + /// only mild truncation before this sampler is recommended. we suggest applying min-p + /// before adaptive-p as the only other active sampler in the chain. + /// + /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) + /// @param decay EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99) + /// @param seed RNG seed + /// + /// ref: https://github.com/ggml-org/llama.cpp/pull/17927 + /// + LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p( + float target, + float decay, + uint32_t seed); + + LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( + int32_t n_vocab, + int32_t n_logit_bias, + const llama_logit_bias * logit_bias); + + // this sampler is meant to be used for fill-in-the-middle infilling + // it's supposed to be used after top_k + top_p sampling + // + // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG + // 2. combine probs of tokens that have the same prefix + // + // example: + // + // - before: + // "hel": 0.5 + // "hell": 0.2 + // "hello": 0.1 + // "dummy": 0.1 + // + // - after: + // "hel": 0.8 + // "dummy": 0.1 + // + // 3. discard non-EOG tokens with low prob + // 4. if no tokens are left -> pick EOT + // + LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab); + + // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise + LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl); + + /// @details Sample and accept a token from the idx-th output of the last evaluation + // + // Shorthand for: + // const auto * logits = llama_get_logits_ith(ctx, idx); + // llama_token_data_array cur_p = { ... init from logits ... }; + // llama_sampler_apply(smpl, &cur_p); + // auto token = cur_p.data[cur_p.selected].id; + // llama_sampler_accept(smpl, token); + // return token; + // Returns the sampled token + LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); + + // TODO: extend in the future + //LLAMA_API void llama_decode_with_sampler(struct llama_context * ctx, struct llama_sampler * smpl, struct llama_batch batch, ...); + + // + // Model split + // + + /// @details Build a split GGUF final path for this chunk. + /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" + // Returns the split_path length. + LLAMA_API int32_t llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int32_t split_no, int32_t split_count); + + /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. + /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" + // Returns the split_prefix length. + LLAMA_API int32_t llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int32_t split_no, int32_t split_count); + + // Print system information + LLAMA_API const char * llama_print_system_info(void); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + // The logger state is global so these functions are NOT thread safe. + LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data); + LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); + + // + // Performance utils + // + // NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements. + // + + struct llama_perf_context_data { + // ms == milliseconds + double t_start_ms; // absolute start time + double t_load_ms; // time needed for loading the model + double t_p_eval_ms; // time needed for processing the prompt + double t_eval_ms; // time needed for generating tokens + + int32_t n_p_eval; // number of prompt tokens + int32_t n_eval; // number of generated tokens + int32_t n_reused; // number of times a ggml compute graph had been reused + }; + + struct llama_perf_sampler_data { + double t_sample_ms; // time needed for sampling in ms + + int32_t n_sample; // number of sampled tokens + }; + + LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx); + LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); + LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); + + // NOTE: the following work only with samplers constructed via llama_sampler_chain_init + LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain); + LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); + LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); + + // + // training + // + + // function that returns whether or not a given tensor contains trainable parameters + typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata); + + // always returns true + LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata); + + struct llama_opt_params { + uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0 + + llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters + void * param_filter_ud; // userdata for determining which tensors contain trainable parameters + + ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters + void * get_opt_pars_ud; // userdata for calculating optimizer parameters + + enum ggml_opt_optimizer_type optimizer_type; + }; + + LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params); + + LLAMA_API void llama_opt_epoch( + struct llama_context * lctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, + ggml_opt_result_t result_eval, + int64_t idata_split, + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval); + +#ifdef __cplusplus +} +#endif + +#endif // LLAMA_H diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/mtmd-helper.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/mtmd-helper.h new file mode 100644 index 00000000..164b7c66 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/mtmd-helper.h @@ -0,0 +1,176 @@ +#ifndef MTMD_HELPER_H +#define MTMD_HELPER_H + +#include "ggml.h" +#include "llama.h" +#include "mtmd.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// +// libmtmd helper functions +// +// Please note that these helpers are not guaranteed to be stable. +// BREAKING CHANGES are expected. +// + +struct mtmd_helper_video; +typedef struct mtmd_helper_video mtmd_helper_video; + +// Set callback for all future logging events. +// If this is not called, or NULL is supplied, everything is output on stderr. +// Note: this also call mtmd_log_set() internally +MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data); + +// Returns true if this build includes video support (MTMD_VIDEO was ON at compile time). +MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx); + +struct mtmd_helper_bitmap_wrapper { + mtmd_bitmap * bitmap; + mtmd_helper_video * video_ctx; +}; + +// helper function to construct a mtmd_bitmap from a file +// it calls mtmd_helper_bitmap_init_from_buf() internally +// returns nullptr on failure +// this function is thread-safe +MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); + +// helper function to construct a mtmd_bitmap from a buffer containing a file +// supported formats: +// image: formats supported by stb_image: jpg, png, bmp, gif, etc. +// audio: formats supported by miniaudio: wav, mp3, flac +// note: +// - for now, video input is only supported via C++ helper functions +// - audio files will be auto-detected based on magic bytes +// - output bitmap will have FNV hash as the ID +// returns nullptr on failure +// this function is thread-safe +MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); + +// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache +MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); + +// helper to count the total position of tokens from a list of chunks, useful to keep track of n_past +// normally, n_pos is equal to n_tokens, but for M-RoPE it is different +MTMD_API llama_pos mtmd_helper_get_n_pos(const mtmd_input_chunks * chunks); + +// helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE +// out_pos must have length == mtmd_helper_get_n_tokens(image) +MTMD_API void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * image, llama_pos pos_0, struct mtmd_decoder_pos * out_pos); + +// helper function that automatically: +// 1. run llama_decode() on text chunks +// 2. run mtmd_encode() on image chunks, then mtmd_get_output_embd() and then llama_decode() +// if any of the mtmd_encode() or llama_decode() calls return non-zero, stop and forward the error +// otherwise, returns 0 on success +// this function is NOT thread-safe +MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunks * chunks, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + bool logits_last, + llama_pos * new_n_past); + +// works like mtmd_helper_eval_chunks(), but only for a single chunk +// this function is NOT thread-safe +MTMD_API int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunk * chunk, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + bool logits_last, + llama_pos * new_n_past); + +// helper function to decode an image whose embeddings have already been calculated +// this helper will handle batching and pre/post decoding setup (for ex. gemma 3 requires non-causal attention) +// ret 0 on success, -1 on chunk not being a valid image chunk, 1 on decode failure +MTMD_API int32_t mtmd_helper_decode_image_chunk(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunk * chunk, + float * encoded_embd, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + llama_pos * new_n_past); + +// +// video input helpers (requires ffmpeg/ffprobe installed on the system) +// the notion of video only exists at the helper level, it is not visible to the core mtmd library +// +// NOTE: this implementation is model-agnostic, it can be used with any vision-capable model +// however, it may not be accurate for some specific models +// (this is expected for now, to keep the implementation simple) +// + +struct mtmd_helper_video_info { + uint32_t width; + uint32_t height; + float fps; // effective fps (fps_target if set, else original video fps) + int32_t n_frames; // estimated total frames at effective fps (-1 if unknown) +}; + +struct mtmd_helper_video_init_params { + float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f + const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH + int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms + // TODO @ngxson : allow "placeholder" bitmap output for counting tokens +}; + +MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void); + +// returns NULL on failure (ffprobe not found, file unreadable, etc.) +MTMD_API mtmd_helper_video * mtmd_helper_video_init( + struct mtmd_context * mctx, + const char * path, + struct mtmd_helper_video_init_params params); + +// Same as mtmd_helper_video_init(), but reads from an in-memory buffer. +// The buffer is copied internally; the caller does not need to keep it alive. +// Note: pipe input is not seekable, so seeking will use output-side seeking +// (ffmpeg decodes and discards frames up to the target position). +MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( + struct mtmd_context * mctx, + const unsigned char * buf, size_t len, + struct mtmd_helper_video_init_params params); +MTMD_API void mtmd_helper_video_free(mtmd_helper_video * ctx); +MTMD_API struct mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx); + +// Read the next item from the video stream; exactly one of out_bitmap or out_text is set per call. +// *out_bitmap - heap-allocated; caller must free with mtmd_bitmap_free() +// *out_text - heap-allocated (always via strdup/malloc); caller must free with free() +// returns 0 on success, -1 on EOF, -2 on error +MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, + mtmd_bitmap ** out_bitmap, + char ** out_text); + +#ifdef __cplusplus +} // extern "C" +#endif + +// +// C++ wrappers +// + +#ifdef __cplusplus +namespace mtmd_helper { + +// video-related C++ wrappers +struct mtmd_helper_video_deleter { + void operator()(mtmd_helper_video * val) { mtmd_helper_video_free(val); } +}; +using video_ptr = std::unique_ptr; + +} // namespace mtmd_helper +#endif + +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/mtmd.h b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/mtmd.h new file mode 100644 index 00000000..a76a6ec2 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Headers/mtmd.h @@ -0,0 +1,387 @@ +#ifndef MTMD_H +#define MTMD_H + +#include "ggml.h" +#include "llama.h" + +#include +#include +#include + +#ifdef __cplusplus +#include +#include +#include +#include +#include +#endif + +/** + * libmtmd: A library for multimodal support in llama.cpp. + * + * WARNING: This API is experimental and subject to many BREAKING CHANGES. + * Issues related to API usage may receive lower priority support. + * + * For the usage, see an example in mtmd-cli.cpp + * + * For contributors: + * - Make sure the C API is aligned with the libllama C API (as in llama.h) + * - Do not include model name (e.g., qwen, gemma) in the API, use generic terms instead + * - Keep the API minimal, do not expose internal details unless necessary + * + * IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated. + * We encourage human contributors to ensure the quality and reliability of the codebase. + */ + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define MTMD_API __declspec(dllexport) +# else +# define MTMD_API __declspec(dllimport) +# endif +# else +# define MTMD_API __attribute__ ((visibility ("default"))) +# endif +#else +# define MTMD_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +enum mtmd_input_chunk_type { + MTMD_INPUT_CHUNK_TYPE_TEXT, + MTMD_INPUT_CHUNK_TYPE_IMAGE, + MTMD_INPUT_CHUNK_TYPE_AUDIO, +}; + +// opaque types +struct mtmd_context; +struct mtmd_bitmap; +struct mtmd_image_tokens; +struct mtmd_input_chunk; +struct mtmd_input_chunks; + +struct mtmd_input_text { + const char * text; + bool add_special; + bool parse_special; +}; + +// +// C API +// + +typedef struct mtmd_context mtmd_context; +typedef struct mtmd_bitmap mtmd_bitmap; +typedef struct mtmd_image_tokens mtmd_image_tokens; +typedef struct mtmd_input_chunk mtmd_input_chunk; +typedef struct mtmd_input_chunks mtmd_input_chunks; +typedef struct mtmd_input_text mtmd_input_text; + +struct mtmd_context_params { + bool use_gpu; + bool print_timings; + int n_threads; + const char * image_marker; // deprecated, use media_marker instead + const char * media_marker; + enum llama_flash_attn_type flash_attn_type; + bool warmup; // whether to run a warmup encode pass after initialization + + // limit number of image tokens, only for vision models with dynamic resolution + int image_min_tokens; // minimum number of tokens for image input (default: read from metadata) + int image_max_tokens; // maximum number of tokens for image input (default: read from metadata) + + // callback function passed over to mtmd proper + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; +}; + +MTMD_API const char * mtmd_default_marker(void); + +MTMD_API struct mtmd_context_params mtmd_context_params_default(void); + +// initialize the mtmd context +// return nullptr on failure +MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname, + const struct llama_model * text_model, + const struct mtmd_context_params ctx_params); + +MTMD_API void mtmd_free(mtmd_context * ctx); + +// whether we need to set non-causal mask before llama_decode +// if chunk is nullptr, we assume the default case where chunk is an image chunk +MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk); + +// whether the current model use M-RoPE for llama_decode +MTMD_API bool mtmd_decode_use_mrope(const mtmd_context * ctx); + +// whether the current model supports vision input +MTMD_API bool mtmd_support_vision(const mtmd_context * ctx); + +// whether the current model supports audio input +MTMD_API bool mtmd_support_audio(const mtmd_context * ctx); + +// get audio sample rate in Hz, for example 16000 for Whisper +// return -1 if audio is not supported +MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); + +// get the current marker string +MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); + +// mtmd_bitmap +// +// if bitmap is image: +// length of data must be nx * ny * 3 +// the data is in RGBRGBRGB... format +// note: some video-capable models (i.e. qwen-vl) can merge consecutive bitmaps +// into one chunk, mtmd_tokenize() will automatically handle this +// if bitmap is audio: +// length of data must be n_samples * sizeof(float) +// the data is in float format (PCM F32) +// +// if data == nullptr: +// the bitmap is considered "empty", and will be treated as a placeholder for counting tokens +// you can pass the bitmap via mtmd_tokenize(), then call mtmd_*_get_n_tokens() to count the tokens +// note: passing a placeholder bitmap to mtmd_encode() will return an error +MTMD_API mtmd_bitmap * mtmd_bitmap_init (uint32_t nx, uint32_t ny, const unsigned char * data); +MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); +MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap); +MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); +MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap); +MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); +MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); +// bitmap ID is optional, but useful for KV cache tracking +// these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data() +MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap); +MTMD_API void mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id); + +// mtmd_bitmap lazy +// +// this is a special bitmap that: +// - does not hold the actual data +// - can be expanded into one or more chunks (either media to text chunks) +// user must provide a callback to fill in the data when mtmd_tokenize() is called +// this is useful for large video inputs: +// - allow reading video frame by frame, without loading the entire video into memory +// - allow tracking the whole video with a single ID (for example, the file hash) + +// set (*out_bitmap) to non-nullptr to emit a bitmap chunk; it will be freed automatically +// set (*out_text) to non-nullptr to emit a text chunk; it must be heap-allocated, null-terminated and will be freed automatically +// either out_bitmap or out_text can be set, but not both +// out_bitmap cannot be another lazy bitmap (no nested lazy allowed) +// return value: +// 0 on success +// -1 on EOF (signal to mtmd_tokenize to move on) +// -2 on error (signal to mtmd_tokenize to abort) +typedef int(* mtmd_bitmap_lazy_callback)( + size_t chunk_idx, + void * user_data, + mtmd_bitmap ** out_bitmap, + char ** out_text); + +MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx, + const char * id, // usually set to file hash + void * user_data, + mtmd_bitmap_lazy_callback callback); + +// mtmd_input_chunks +// +// this is simply a list of mtmd_input_chunk +// the elements can only be populated via mtmd_tokenize() +MTMD_API mtmd_input_chunks * mtmd_input_chunks_init(void); +MTMD_API size_t mtmd_input_chunks_size(const mtmd_input_chunks * chunks); +MTMD_API const mtmd_input_chunk * mtmd_input_chunks_get (const mtmd_input_chunks * chunks, size_t idx); +MTMD_API void mtmd_input_chunks_free(mtmd_input_chunks * chunks); + +// mtmd_input_chunk +// +// the instance will be constructed via mtmd_tokenize() +// it will be freed along with mtmd_input_chunks +MTMD_API enum mtmd_input_chunk_type mtmd_input_chunk_get_type (const mtmd_input_chunk * chunk); +MTMD_API const llama_token * mtmd_input_chunk_get_tokens_text (const mtmd_input_chunk * chunk, size_t * n_tokens_output); +MTMD_API const mtmd_image_tokens * mtmd_input_chunk_get_tokens_image(const mtmd_input_chunk * chunk); +MTMD_API size_t mtmd_input_chunk_get_n_tokens (const mtmd_input_chunk * chunk); +// returns nullptr for ID on text chunk +MTMD_API const char * mtmd_input_chunk_get_id (const mtmd_input_chunk * chunk); +// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise) +MTMD_API llama_pos mtmd_input_chunk_get_n_pos (const mtmd_input_chunk * chunk); + +// in case you want to use custom logic to handle the chunk (i.e. KV cache management) +// you can move the chunk ownership to your own code by copying it +// remember to free the chunk when you are done with it +MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk); +MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk); + + +// mtmd_image_tokens +// +// the instance will be constructed via mtmd_tokenize() +// it will be freed along with mtmd_input_chunk +MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); // TODO: deprecate +MTMD_API const char * mtmd_image_tokens_get_id (const mtmd_image_tokens * image_tokens); // TODO: deprecate +// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise) +MTMD_API llama_pos mtmd_image_tokens_get_n_pos (const mtmd_image_tokens * image_tokens); // TODO: deprecate + +DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens), + "use mtmd_image_tokens_get_decoder_pos() instead"); +DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens), + "use mtmd_image_tokens_get_decoder_pos() instead"); + +struct mtmd_decoder_pos { + uint32_t t; + uint32_t x; + uint32_t y; + uint32_t z; // unused for now, reserved for future use +}; +// get position for decoder attention, to be used by M-RoPE models +// i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 +// pos_0 is the absolute position of the first token +// return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position) +MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i); + +// tokenize an input text prompt and a list of bitmaps (images/audio) +// the prompt must have the input image marker (default: "<__media__>") in it +// the default marker is defined by mtmd_default_marker() +// the marker will be replaced with the image/audio chunk +// for example: +// "here is an image: <__media__>\ndescribe it in detail." +// this will gives 3 chunks: +// 1. "here is an image: " +// 2. (image/audio tokens) +// 3. "\ndescribe it in detail." +// number of bitmaps must be equal to the number of markers in the prompt +// this function is thread-safe (shared ctx) +// return values: +// 0 on success +// 1 on number of bitmaps not matching the number of markers +// 2 on image preprocessing error +MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, + mtmd_input_chunks * output, + const mtmd_input_text * text, + const mtmd_bitmap ** bitmaps, + size_t n_bitmaps); + +// returns 0 on success +// TODO: deprecate +MTMD_API int32_t mtmd_encode(mtmd_context * ctx, + const mtmd_image_tokens * image_tokens); + +// returns 0 on success +MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx, + const mtmd_input_chunk * chunk); + +// get output embeddings from the last encode pass +// the reading size (in bytes) is equal to: +// llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float) +MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx); + +// Set callback for all future logging events. +// If this is not called, or NULL is supplied, everything is output on stderr. +MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); + +// EXPERIMENTAL API to get mmproj's capabilities without initializing the full context +// This is only intended to be used by llama-server, breaking changes is expected +struct mtmd_caps { + bool inp_vision; + bool inp_audio; +}; +MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname); + +///////////////////////////////////////// + +// test function, to be used in test-mtmd-c-api.c +MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +// Get memory usage of the current model in bytes, per backend device +// Note: this is an unstable API, used internally by fit_params; it WILL be removed or changed without deprecation +#ifdef __cplusplus +MTMD_API std::map mtmd_get_memory_usage( + const char * mmproj_fname, + struct mtmd_context_params ctx_params); +#endif + +// +// C++ wrappers +// + +#ifdef __cplusplus + +namespace mtmd { + +struct mtmd_context_deleter { + void operator()(mtmd_context * val) { mtmd_free(val); } +}; +using context_ptr = std::unique_ptr; + +struct mtmd_bitmap_deleter { + void operator()(mtmd_bitmap * val) { mtmd_bitmap_free(val); } +}; +using bitmap_ptr = std::unique_ptr; + +struct mtmd_input_chunks_deleter { + void operator()(mtmd_input_chunks * val) { mtmd_input_chunks_free(val); } +}; +using input_chunks_ptr = std::unique_ptr; + +struct mtmd_input_chunk_deleter { + void operator()(mtmd_input_chunk * val) { mtmd_input_chunk_free(val); } +}; +using input_chunk_ptr = std::unique_ptr; + +struct bitmap { + bitmap_ptr ptr; + bitmap() : ptr(nullptr) {} + bitmap(mtmd_bitmap * bitmap) : ptr(bitmap) {} + bitmap(bitmap && other) noexcept : ptr(std::move(other.ptr)) {} + bitmap(uint32_t nx, uint32_t ny, const unsigned char * data) { + ptr.reset(mtmd_bitmap_init(nx, ny, data)); + } + ~bitmap() = default; + uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } + uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } + const unsigned char * data() const { return mtmd_bitmap_get_data(ptr.get()); } + size_t n_bytes() const { return mtmd_bitmap_get_n_bytes(ptr.get()); } + std::string id() const { return mtmd_bitmap_get_id(ptr.get()); } + void set_id(const char * id) const { mtmd_bitmap_set_id(ptr.get(), id); } +}; + +struct bitmaps { + std::vector entries; + ~bitmaps() = default; + // return list of pointers to mtmd_bitmap + // example: + // auto bitmaps_c_ptr = bitmaps.c_ptr(); + // int32_t res = mtmd_tokenize(... bitmaps_c_ptr.data(), bitmaps_c_ptr.size()); + std::vector c_ptr() { + std::vector res(entries.size()); + for (size_t i = 0; i < entries.size(); i++) { + res[i] = entries[i].ptr.get(); + } + return res; + } +}; + +struct input_chunks { + input_chunks_ptr ptr; + input_chunks() = default; + input_chunks(mtmd_input_chunks * chunks) : ptr(chunks) {} + ~input_chunks() = default; + size_t size() const { return mtmd_input_chunks_size(ptr.get()); } + const mtmd_input_chunk * operator[](size_t idx) const { + return mtmd_input_chunks_get(ptr.get(), idx); + } +}; + +} // namespace mtmd + +#endif + +#endif diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Info.plist b/ios/Llama.xcframework/ios-arm64/Llama.framework/Info.plist new file mode 100644 index 00000000..b5f25cf6 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Info.plist @@ -0,0 +1,16 @@ + + + + + CFBundleDevelopmentRegionen + CFBundleExecutableLlama + CFBundleIdentifierorg.ggml.llama + CFBundleInfoDictionaryVersion6.0 + CFBundleNameLlama + CFBundlePackageTypeFMWK + CFBundleShortVersionString0.9.0 + CFBundleVersion1 + MinimumOSVersion14.0 + + diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Llama b/ios/Llama.xcframework/ios-arm64/Llama.framework/Llama new file mode 100755 index 00000000..3e698bdf Binary files /dev/null and b/ios/Llama.xcframework/ios-arm64/Llama.framework/Llama differ diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/Modules/module.modulemap b/ios/Llama.xcframework/ios-arm64/Llama.framework/Modules/module.modulemap new file mode 100644 index 00000000..4d610a57 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/Modules/module.modulemap @@ -0,0 +1,5 @@ +framework module Llama { + umbrella header "llama.h" + export * + module * { export * } +} diff --git a/ios/Llama.xcframework/ios-arm64/Llama.framework/_CodeSignature/CodeResources b/ios/Llama.xcframework/ios-arm64/Llama.framework/_CodeSignature/CodeResources new file mode 100644 index 00000000..1e94d0ff --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64/Llama.framework/_CodeSignature/CodeResources @@ -0,0 +1,223 @@ + + + + + files + + Headers/ggml-alloc.h + + I0o1yNqMayC5htF3onYJoQOrjpE= + + Headers/ggml-backend.h + + BXJOIu9ZRY0KkU+9j4gvnFF1lfs= + + Headers/ggml-cpu.h + + hO3RnG3fEsoqT2LnyHNIkD5ylQA= + + Headers/ggml-metal.h + + OPKWzK2BQxc8uydhDsIS9Ot2qCE= + + Headers/ggml-opt.h + + f9+8P3HT1FQTRJZGtBT0sEIv9VE= + + Headers/ggml.h + + mISflhL9UXaRyMWayFkDfFedOrc= + + Headers/gguf.h + + VLK4ktzJlOdm9759ZJ+Y74si958= + + Headers/llama.h + + P79ZX6ISVVhv6WNspP0FAw0T6lo= + + Headers/mtmd-helper.h + + zzc0rHtTY+FMBCdBGNeaHoKLNX4= + + Headers/mtmd.h + + DrL4/IcJRdtDmGKglKnPuh8Tjk0= + + Info.plist + + iEX8HYaevfNCkb+BVWvj2hW/zUQ= + + Modules/module.modulemap + + GLxr401eUfp6oByYq4y5zddmobY= + + + files2 + + Headers/ggml-alloc.h + + hash2 + + lOTNBpuTE7LOs12s7JAZgeC7R42LsxA1txJr4JGZjCM= + + + Headers/ggml-backend.h + + hash2 + + piDoFbQ6RMxy1fIWYpo6kZgDNbYbw362stCBM2jDcE8= + + + Headers/ggml-cpu.h + + hash2 + + Gq/pflduo4wNpXUX+ySSlV0LacGnmYQkgfwGnW2dKO8= + + + Headers/ggml-metal.h + + hash2 + + Mi82zTDz6eeq17W5vGMHgBL9C3cGrE4k9XIbYE89iYA= + + + Headers/ggml-opt.h + + hash2 + + NYbeG8ipNLXHIznitpN7BkHo8Um1EiMeZm9n3gc27qI= + + + Headers/ggml.h + + hash2 + + 7fzmpz5ifSIN2K68xtYgkXVbEAYUrX2lMpKlKLGwB1A= + + + Headers/gguf.h + + hash2 + + Ldsnalvs50NDMWCthjJ5RzQxydTBcdRov4YKPNo/vz8= + + + Headers/llama.h + + hash2 + + VrCntN56IKB+D6+KWox7hbhvo6E2c4XcmMr+Gm3HrUY= + + + Headers/mtmd-helper.h + + hash2 + + U+xT7AinBJ5PF493agD+pZ7wGPsLuw8sgAH17wfcCxk= + + + Headers/mtmd.h + + hash2 + + xX/4y6LQ5ZtrHg6Q/snhiHXMUmi65nNvxIuii2TPnPE= + + + Modules/module.modulemap + + hash2 + + YrZrr8fQjLcP/nkdW+LigUSHc1OnNtYwwRm5qRmrFFw= + + + + rules + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-alloc.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-alloc.h new file mode 100644 index 00000000..a7926a21 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-alloc.h @@ -0,0 +1,86 @@ +#pragma once + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; +typedef struct ggml_backend_buffer * ggml_backend_buffer_t; +typedef struct ggml_backend * ggml_backend_t; + +// Tensor allocator +struct ggml_tallocr { + ggml_backend_buffer_t buffer; + void * base; + size_t alignment; + size_t offset; +}; + +GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer); +GGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor); + +// Graph allocator +/* + Example usage: + ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + + // optional: create a worst-case graph and reserve the buffers to avoid reallocations + ggml_gallocr_reserve(galloc, build_graph(max_batch)); + + // allocate the graph + struct ggml_cgraph * graph = build_graph(batch); + ggml_gallocr_alloc_graph(galloc, graph); + + printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0)); + + // evaluate the graph + ggml_backend_graph_compute(backend, graph); +*/ + +// special tensor flags for use with the graph allocator: +// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses +// ggml_set_output(): output tensors are never freed and never overwritten + +typedef struct ggml_gallocr * ggml_gallocr_t; + +GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft); +GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs); +GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc); + +// pre-allocate buffers from a measure graph - does not allocate or modify the graph +// call with a worst-case graph to avoid buffer reallocations +// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed +// returns false if the buffer allocation failed +// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes +GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph); +GGML_API void ggml_gallocr_reserve_n_size( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids, + size_t * sizes); +GGML_API bool ggml_gallocr_reserve_n( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids); + +// automatic reallocation if the topology changes when using a single buffer +// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers) +GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph); + +GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id); + +// Utils +// Create a buffer and allocate all the tensors in a ggml_context +// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft +// ggml_backend_alloc_ctx_tensors_from_buft returns NULL on failure or if all tensors in ctx are already allocated or zero-sized +GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-backend.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-backend.h new file mode 100644 index 00000000..2924fdbe --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-backend.h @@ -0,0 +1,435 @@ +#pragma once + +#include "ggml.h" +#include "ggml-alloc.h" + +#ifdef GGML_BACKEND_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BACKEND_BUILD +# define GGML_BACKEND_API __declspec(dllexport) extern +# else +# define GGML_BACKEND_API __declspec(dllimport) extern +# endif +# else +# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_BACKEND_API extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; + typedef struct ggml_backend_buffer * ggml_backend_buffer_t; + typedef struct ggml_backend_event * ggml_backend_event_t; + typedef struct ggml_backend * ggml_backend_t; + typedef void * ggml_backend_graph_plan_t; + typedef struct ggml_backend_reg * ggml_backend_reg_t; + typedef struct ggml_backend_device * ggml_backend_dev_t; + + + // + // Backend buffer type + // + + GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); + GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); + GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft); + + // + // Backend buffer + // + + enum ggml_backend_buffer_usage { + GGML_BACKEND_BUFFER_USAGE_ANY = 0, + GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1, + GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2, + }; + + GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); + GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); + GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor); + GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); + GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer); + GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); + + // tensor copy between different backends + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); + + // + // Backend (stream) + // + + GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend); + GGML_API const char * ggml_backend_name(ggml_backend_t backend); + GGML_API void ggml_backend_free(ggml_backend_t backend); + + GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend); + GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size); + GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); + GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); + + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + + // "offset" refers to the offset in tensor->data for setting/getting data + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + + GGML_API void ggml_backend_synchronize(ggml_backend_t backend); + + GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + + GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); + + // NOTE: will be removed, use device version instead + GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); + GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op); + + // asynchronous copy + // the copy is performed after all the currently queued operations in backend_src + // backend_dst will wait for the copy to complete before performing other operations + // automatic fallback to sync copy if async is not supported + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); + + GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); + + // + // Events + // + + GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device); + GGML_API void ggml_backend_event_free(ggml_backend_event_t event); + GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend); + GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event); + GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event); + + // + // Backend device + // + + enum ggml_backend_dev_type { + // CPU device using system memory + GGML_BACKEND_DEVICE_TYPE_CPU, + // GPU device using dedicated memory + GGML_BACKEND_DEVICE_TYPE_GPU, + // integrated GPU device using host memory + GGML_BACKEND_DEVICE_TYPE_IGPU, + // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, + }; + + // functionality supported by the device + struct ggml_backend_dev_caps { + // asynchronous operations + bool async; + // pinned host buffer + bool host_buffer; + // creating buffers from host ptr + bool buffer_from_host_ptr; + // event synchronization + bool events; + }; + + // all the device properties + struct ggml_backend_dev_props { + // device name + const char * name; + // device description + const char * description; + // device free memory in bytes + size_t memory_free; + // device total memory in bytes + size_t memory_total; + // device type + enum ggml_backend_dev_type type; + // device id + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") + // if the id is unknown, this should be NULL + const char * device_id; + // device capabilities + struct ggml_backend_dev_caps caps; + }; + + GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device); + GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total); + GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props); + GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device); + GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size); + + GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + + // + // Backend (reg) + // + + GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg); + GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg); + GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index); + GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name); + + // Common functions that may be obtained using ggml_backend_reg_get_proc_address + + // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) + typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); + typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); + + // Split buffer type for tensor parallelism (old) + typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); + // Set the number of threads for the backend + typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); + // Get additional buffer types provided by the device (returns a NULL-terminated array) + typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device); + // Set the abort callback for the backend + typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data); + // Get a list of feature flags supported by the backend (returns a NULL-terminated array) + struct ggml_backend_feature { + const char * name; + const char * value; + }; + typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg); + + // + // Backend registry + // + + GGML_API void ggml_backend_register(ggml_backend_reg_t reg); + + GGML_API void ggml_backend_device_register(ggml_backend_dev_t device); + + // Backend (reg) enumeration + GGML_API size_t ggml_backend_reg_count(void); + GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index); + GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name); + + // Device enumeration + GGML_API size_t ggml_backend_dev_count(void); + GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type); + + // Direct backend (stream) initialization + // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params) + GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params) + GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL) + GGML_API ggml_backend_t ggml_backend_init_best(void); + + // Load a backend from a dynamic library and register it + GGML_API ggml_backend_reg_t ggml_backend_load(const char * path); + // Unload a backend if loaded dynamically and unregister it + GGML_API void ggml_backend_unload(ggml_backend_reg_t reg); + // Load all known backends from dynamic libraries + GGML_API void ggml_backend_load_all(void); + GGML_API void ggml_backend_load_all_from_path(const char * dir_path); + + // + // Backend scheduler + // + + // The backend scheduler allows for multiple backend devices to be used together + // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends + // The backends are selected based on: + // - the backend that supports the operation + // - the location of the pre-allocated tensors (e.g. the weights) + /* + Example usage: + + // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned + // preferably to run on the same backend as the buffer + ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true); + + // initialize buffers from a max size graph (optional) + reserve_graph = build_graph(sched, max_batch_size); + + // manually assign nodes to a backend (optional, should not be needed in most cases) + struct ggml_tensor * node = ggml_mul_mat(ctx, ...); + ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu); + + ggml_backend_sched_reserve(sched, reserve_graph); + + // compute + graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation + for (int i = 0; i < 10; ++i) { + ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically + } + + // if there are graph inputs: + graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called) + ggml_backend_sched_reset(sched); // clear the allocation of the previous graph + ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it + ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors + ggml_backend_sched_graph_compute(sched, graph); // execute the graph + + // as an alternative to the above it is also possible to assign the inputs to a dedicated context and + // allocate them statically via ggml_backend_alloc_ctx_tensors + } + */ + + typedef struct ggml_backend_sched * ggml_backend_sched_t; + + // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback) + // when ask == true, the scheduler wants to know if the user wants to observe this node + // this allows the scheduler to batch nodes together in order to evaluate them in a single call + // + // when ask == false, the scheduler is passing the node tensor to the user for observation + // if the user returns false, the scheduler will cancel the graph compute + // + typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); + + // Initialize a backend scheduler, backends with low index are given priority over backends with high index + GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload); + GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); + + // Initialize backend buffers from a measure graph + GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes); + GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success + + GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched); + GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i); + + // Get the number of splits of the last graph + GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); + GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched); + + GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend); + GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + + GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); + + // Split graph without allocating it + GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + + // Allocate and compute graph on the backend scheduler + GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success + GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched); + + // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph. + // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers. + // The correct way to use this API is to discard the deallocated tensors and create new ones. + GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); + + // Set a callback to be called for each resulting node during graph compute + GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + + // + // Meta backend + // + +#define GGML_BACKEND_META_MAX_DEVICES 16 + + enum ggml_backend_meta_split_axis { + // tensor split by tensor dimensions: + GGML_BACKEND_SPLIT_AXIS_0 = 0, + GGML_BACKEND_SPLIT_AXIS_1 = 1, + GGML_BACKEND_SPLIT_AXIS_2 = 2, + GGML_BACKEND_SPLIT_AXIS_3 = 3, + + GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends + GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum + + // for internal bookkeeping only: + GGML_BACKEND_SPLIT_AXIS_NONE = 98, + GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99, + }; + GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis); + + struct ggml_backend_meta_split_state { + enum ggml_backend_meta_split_axis axis; + + // for tensors with axis >= 0 && axis < GGML_MAX_DIMS: + // - each device has a slice of the tensor along the split axis + // - most tensors have n_segments == 1 and a contiguous slice of the tensor data + // - some tensors have an inhomogenenous data layout along the split axis, + // those tensors are divided into segments which are each individually split across devices + // - ne has one entry per segment and device and that segment repeats nr times, + // in total when accounting for repetitions the segments add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0_r0, seg0_dev1_r0, seg0_dev0_r1, seg0_dev1_r1, seg1_dev0_r0, seg1_dev1_r0], + // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V, + // the Q matrix can be larger than the K and V matrices so this can either be expressed as 3 segments or as 2 segments + // where the segment for K/V repeats twice + int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t nr[16]; + uint32_t n_segments; + }; + + // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible: + typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata); + + // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this: + // TODO: this looks a bit strange - a backend API creates a device. I think we should try + // express this as a backend registry functionality instead + GGML_API ggml_backend_dev_t ggml_backend_meta_device( + ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud); + + // + // Utils + // + + struct ggml_backend_graph_copy { + ggml_backend_buffer_t buffer; + struct ggml_context * ctx_allocated; + struct ggml_context * ctx_unallocated; + struct ggml_cgraph * graph; + }; + + // Copy a graph to a different backend + GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph); + GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy); + + typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data); + + // Compare the output of two backends + GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes); + + // Tensor initialization + GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr); + GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor); + + // CPU buffer types are always available + GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); + GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-cpu.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-cpu.h new file mode 100644 index 00000000..e3e067c9 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-cpu.h @@ -0,0 +1,151 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + + // the compute plan that needs to be prepared for ggml_graph_compute() + // since https://github.com/ggml-org/ggml/issues/287 + struct ggml_cplan { + size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()` + uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()` + + int n_threads; + struct ggml_threadpool * threadpool; + + // abort ggml_graph_compute when true + ggml_abort_callback abort_callback; + void * abort_callback_data; + + // use only reference implementations + bool use_ref; + }; + + // numa strategies + enum ggml_numa_strategy { + GGML_NUMA_STRATEGY_DISABLED = 0, + GGML_NUMA_STRATEGY_DISTRIBUTE = 1, + GGML_NUMA_STRATEGY_ISOLATE = 2, + GGML_NUMA_STRATEGY_NUMACTL = 3, + GGML_NUMA_STRATEGY_MIRROR = 4, + GGML_NUMA_STRATEGY_COUNT + }; + + GGML_BACKEND_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems + GGML_BACKEND_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node + + GGML_BACKEND_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value); + GGML_BACKEND_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value); + + GGML_BACKEND_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value); + GGML_BACKEND_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value); + + GGML_BACKEND_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i); + GGML_BACKEND_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value); + + GGML_BACKEND_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3); + GGML_BACKEND_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value); + + GGML_BACKEND_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i); + GGML_BACKEND_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value); + + GGML_BACKEND_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3); + GGML_BACKEND_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value); + + GGML_BACKEND_API struct ggml_threadpool * ggml_threadpool_new (struct ggml_threadpool_params * params); + GGML_BACKEND_API void ggml_threadpool_free (struct ggml_threadpool * threadpool); + GGML_BACKEND_API int ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool); + GGML_BACKEND_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool); + GGML_BACKEND_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool); + + // ggml_graph_plan() has to be called before ggml_graph_compute() + // when plan.work_size > 0, caller must allocate memory for plan.work_data + GGML_BACKEND_API struct ggml_cplan ggml_graph_plan( + const struct ggml_cgraph * cgraph, + int n_threads, /* = GGML_DEFAULT_N_THREADS */ + struct ggml_threadpool * threadpool /* = NULL */ ); + GGML_BACKEND_API enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan); + + // same as ggml_graph_compute() but the work data is allocated as a part of the context + // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data + GGML_BACKEND_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads); + + // + // system info + // + + // x86 + GGML_BACKEND_API int ggml_cpu_has_sse3 (void); + GGML_BACKEND_API int ggml_cpu_has_ssse3 (void); + GGML_BACKEND_API int ggml_cpu_has_avx (void); + GGML_BACKEND_API int ggml_cpu_has_avx_vnni (void); + GGML_BACKEND_API int ggml_cpu_has_avx2 (void); + GGML_BACKEND_API int ggml_cpu_has_bmi2 (void); + GGML_BACKEND_API int ggml_cpu_has_f16c (void); + GGML_BACKEND_API int ggml_cpu_has_fma (void); + GGML_BACKEND_API int ggml_cpu_has_avx512 (void); + GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void); + GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void); + GGML_BACKEND_API int ggml_cpu_has_avx512_bf16(void); + GGML_BACKEND_API int ggml_cpu_has_amx_int8 (void); + // ARM + GGML_BACKEND_API int ggml_cpu_has_neon (void); + GGML_BACKEND_API int ggml_cpu_has_arm_fma (void); + GGML_BACKEND_API int ggml_cpu_has_fp16_va (void); + GGML_BACKEND_API int ggml_cpu_has_dotprod (void); + GGML_BACKEND_API int ggml_cpu_has_matmul_int8(void); + GGML_BACKEND_API int ggml_cpu_has_sve (void); + GGML_BACKEND_API int ggml_cpu_get_sve_cnt (void); // sve vector length in bytes + GGML_BACKEND_API int ggml_cpu_has_sme (void); + // other + GGML_BACKEND_API int ggml_cpu_has_riscv_v (void); + GGML_BACKEND_API int ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytes + GGML_BACKEND_API int ggml_cpu_has_vsx (void); + GGML_BACKEND_API int ggml_cpu_has_vxe (void); + GGML_BACKEND_API int ggml_cpu_has_wasm_simd (void); + GGML_BACKEND_API int ggml_cpu_has_llamafile (void); + + // Internal types and functions exposed for tests and benchmarks + + typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx, + const void * GGML_RESTRICT y, size_t by, int nrc); + + struct ggml_type_traits_cpu { + ggml_from_float_t from_float; + ggml_vec_dot_t vec_dot; + enum ggml_type vec_dot_type; + int64_t nrows; // number of rows to process simultaneously + }; + + GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type); + + GGML_BACKEND_API void ggml_cpu_init(void); + + // + // CPU backend + // + + GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void); + + GGML_BACKEND_API bool ggml_backend_is_cpu (ggml_backend_t backend); + GGML_BACKEND_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads); + GGML_BACKEND_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool); + GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data); + + GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref); + + GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void); + + GGML_BACKEND_API void ggml_cpu_fp32_to_fp32(const float *, float *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_i32 (const float *, int32_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_fp16(const float *, ggml_fp16_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp16_to_fp32(const ggml_fp16_t *, float *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_bf16(const float *, ggml_bf16_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_bf16_to_fp32(const ggml_bf16_t *, float *, int64_t); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-metal.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-metal.h new file mode 100644 index 00000000..433838f0 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-metal.h @@ -0,0 +1,61 @@ +// Note: this description is outdated +// +// An interface allowing to compute ggml_cgraph with Metal +// +// This is a fully functional interface that extends ggml with GPU support for Apple devices. +// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.) +// +// How it works? +// +// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this +// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you +// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.) +// +// You only need to make sure that all memory buffers that you used during the graph creation +// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is +// used during the graph evaluation to determine the arguments of the compute kernels. +// +// Synchronization between device and host memory (for example for input and output tensors) +// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions. +// + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include + +struct ggml_tensor; +struct ggml_cgraph; + +#ifdef __cplusplus +extern "C" { +#endif + +// +// backend API +// user-code should use only these functions +// + +// TODO: remove in the future +GGML_BACKEND_API ggml_backend_t ggml_backend_metal_init(void); + +GGML_BACKEND_API bool ggml_backend_is_metal(ggml_backend_t backend); + +GGML_BACKEND_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data); + +// helper to check if the device supports a specific family +// ideally, the user code should be doing these checks +// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf +GGML_BACKEND_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family); + +// capture all command buffers committed the next time `ggml_backend_graph_compute` is called +GGML_BACKEND_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_metal_reg(void); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-opt.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-opt.h new file mode 100644 index 00000000..1c2ed79b --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml-opt.h @@ -0,0 +1,256 @@ +// This file contains functionality for training models using GGML. +// It is not strictly needed vs. just vanilla GGML but it provides a more high-level interface for common needs such as datasets. +// At the bottom of this file especially there are relatively high-level functions that are suitable use or adaptation in user code. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + struct ggml_opt_dataset; + struct ggml_opt_context; + struct ggml_opt_result; + + typedef struct ggml_opt_dataset * ggml_opt_dataset_t; + typedef struct ggml_opt_context * ggml_opt_context_t; + typedef struct ggml_opt_result * ggml_opt_result_t; + + // ====== Loss ====== + + // built-in loss types, i.e. the built-in quantities minimized by the optimizer + // custom loss types can be defined via mean or sum which simply reduce the outputs for all datapoints to a single value + enum ggml_opt_loss_type { + GGML_OPT_LOSS_TYPE_MEAN, + GGML_OPT_LOSS_TYPE_SUM, + GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, + GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR, + }; + + // ====== Dataset ====== + + GGML_API ggml_opt_dataset_t ggml_opt_dataset_init( + enum ggml_type type_data, // the type for the internal data tensor + enum ggml_type type_label, // the type for the internal labels tensor + int64_t ne_datapoint, // number of elements per datapoint + int64_t ne_label, // number of elements per label + int64_t ndata, // total number of datapoints/labels + int64_t ndata_shard); // number of datapoints/labels per shard (unit at which the dataset is shuffled/copied) + GGML_API void ggml_opt_dataset_free(ggml_opt_dataset_t dataset); + + // get underlying tensors that store the data + GGML_API int64_t ggml_opt_dataset_ndata (ggml_opt_dataset_t dataset); + GGML_API struct ggml_tensor * ggml_opt_dataset_data (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata] + GGML_API struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset); // shape = [nd_label, ndata] + + // shuffle idata first datapoints from dataset with RNG from opt_ctx, shuffle all datapoints if idata is negative + GGML_API void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata); + + // get batch at position ibatch from dataset and copy the data to data_batch and labels_batch + GGML_API void ggml_opt_dataset_get_batch( + ggml_opt_dataset_t dataset, + struct ggml_tensor * data_batch, // shape = [ne_datapoint, ndata_batch] + struct ggml_tensor * labels_batch, // shape = [ne_label, ndata_batch] + int64_t ibatch); + GGML_API void ggml_opt_dataset_get_batch_host( + ggml_opt_dataset_t dataset, + void * data_batch, + size_t nb_data_batch, + void * labels_batch, + int64_t ibatch); + + // ====== Model / Context ====== + + enum ggml_opt_build_type { + GGML_OPT_BUILD_TYPE_FORWARD = 10, + GGML_OPT_BUILD_TYPE_GRAD = 20, + GGML_OPT_BUILD_TYPE_OPT = 30, + }; + + enum ggml_opt_optimizer_type { + GGML_OPT_OPTIMIZER_TYPE_ADAMW, + GGML_OPT_OPTIMIZER_TYPE_SGD, + + GGML_OPT_OPTIMIZER_TYPE_COUNT + }; + + // parameters that control which optimizer is used and how said optimizer tries to find the minimal loss + struct ggml_opt_optimizer_params { + struct { + float alpha; // learning rate + float beta1; // first AdamW momentum + float beta2; // second AdamW momentum + float eps; // epsilon for numerical stability + float wd; // weight decay - 0.0f to disable + } adamw; + struct { + float alpha; // learning rate + float wd; // weight decay + } sgd; + }; + + // callback to calculate optimizer parameters prior to a backward pass + // userdata can be used to pass arbitrary data + typedef struct ggml_opt_optimizer_params (*ggml_opt_get_optimizer_params)(void * userdata); + + // returns the default optimizer params (constant, hard-coded values) + // userdata is not used + GGML_API struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata); + + // casts userdata to ggml_opt_optimizer_params and returns it + GGML_API struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata); + + // parameters for initializing a new optimization context + struct ggml_opt_params { + ggml_backend_sched_t backend_sched; // defines which backends are used to construct the compute graphs + + // by default the forward graph needs to be reconstructed for each eval + // if ctx_compute, inputs, and outputs are set the graphs are instead allocated statically + struct ggml_context * ctx_compute; + struct ggml_tensor * inputs; + struct ggml_tensor * outputs; + + enum ggml_opt_loss_type loss_type; + enum ggml_opt_build_type build_type; + + int32_t opt_period; // after how many gradient accumulation steps an optimizer step should be done + + ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters + void * get_opt_pars_ud; // userdata for calculating optimizer parameters + + // only GGML_OPT_OPTIMIZER_TYPE_ADAMW needs m, v momenta per parameter tensor + enum ggml_opt_optimizer_type optimizer; + }; + + // get parameters for an optimization context with defaults set where possible + // parameters for which no sensible defaults exist are supplied as arguments to this function + GGML_API struct ggml_opt_params ggml_opt_default_params( + ggml_backend_sched_t backend_sched, + enum ggml_opt_loss_type loss_type); + + GGML_API ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params); + GGML_API void ggml_opt_free(ggml_opt_context_t opt_ctx); + + // set gradients to zero, initialize loss, and optionally reset the optimizer + GGML_API void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer); + + GGML_API bool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx); // whether the graphs are allocated_statically + + // get underlying tensors that store data + // if not using static graphs these pointers become invalid with the next call to ggml_opt_alloc + GGML_API struct ggml_tensor * ggml_opt_inputs( ggml_opt_context_t opt_ctx); // forward graph input tensor + GGML_API struct ggml_tensor * ggml_opt_outputs( ggml_opt_context_t opt_ctx); // forward graph output tensor + GGML_API struct ggml_tensor * ggml_opt_labels( ggml_opt_context_t opt_ctx); // labels to compare outputs against + GGML_API struct ggml_tensor * ggml_opt_loss( ggml_opt_context_t opt_ctx); // scalar tensor that contains the loss + GGML_API struct ggml_tensor * ggml_opt_pred( ggml_opt_context_t opt_ctx); // predictions made by outputs + GGML_API struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx); // number of matching predictions between outputs and labels + + // get the gradient accumulator for a node from the forward graph + GGML_API struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node); + + GGML_API enum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t); //TODO consistent naming scheme + + GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type); + + // ====== Optimization Result ====== + + GGML_API ggml_opt_result_t ggml_opt_result_init(void); + GGML_API void ggml_opt_result_free(ggml_opt_result_t result); + GGML_API void ggml_opt_result_reset(ggml_opt_result_t result); + + // get data from result, uncertainties are optional and can be ignored by passing NULL + GGML_API void ggml_opt_result_ndata( ggml_opt_result_t result, int64_t * ndata); // writes 1 value, number of datapoints + GGML_API void ggml_opt_result_loss( ggml_opt_result_t result, double * loss, double * unc); // writes 1 value + GGML_API void ggml_opt_result_pred( ggml_opt_result_t result, int32_t * pred); // writes ndata values + GGML_API void ggml_opt_result_accuracy(ggml_opt_result_t result, double * accuracy, double * unc); // writes 1 value + + // ====== Computation ====== + + // if not using static graphs, this function must be called prior to ggml_opt_alloc + GGML_API void ggml_opt_prepare_alloc( + ggml_opt_context_t opt_ctx, + struct ggml_context * ctx_compute, + struct ggml_cgraph * gf, + struct ggml_tensor * inputs, + struct ggml_tensor * outputs); + + // allocate the next graph for evaluation, either forward or forward + backward + // must be called exactly once prior to calling ggml_opt_eval + GGML_API void ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward); + + // do forward pass, increment result if not NULL, do backward pass if allocated + GGML_API void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result); + + // ############################################################################ + // ## The high-level functions start here. They do not depend on any private ## + // ## functions or structs and can be copied to and adapted for user code. ## + // ############################################################################ + + // ====== Intended Usage ====== + // + // 1. Select the appropriate loss for your problem. + // 2. Create a dataset and set the data for the "data" tensor. Also set the "labels" tensor if your loss needs them. + // Setting the shard size to 1 will be fine, it's the granularity with which data is shuffled/loaded (bigger values are faster). + // 3. Create a GGML graph for your model with no_alloc == true. Use two separate contexts for the tensors. + // The first context should contain the model parameters and inputs and be allocated statically in user code. + // The second context should contain all other tensors and will be (re)allocated automatically. + // Due to this automated allocation the data of the second context is not defined when accessed in user code. + // Note that the second dimension of the inputs/outputs are interpreted as the number of datapoints in those tensors. + // 4. Call ggml_opt_fit. If you need more control you can use ggml_opt_epoch instead. + + // signature for a callback while evaluating opt_ctx on dataset, called after an evaluation + typedef void (*ggml_opt_epoch_callback)( + bool train, // true after training evaluation, false after validation evaluation + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, // result associated with the dataset subsection + int64_t ibatch, // number of batches that have been evaluated so far + int64_t ibatch_max, // total number of batches in this dataset subsection + int64_t t_start_us); // time at which the evaluation on the dataset subsection was started + + // do training on front of dataset, do evaluation only on back of dataset + GGML_API void ggml_opt_epoch( + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, // result to increment during training, ignored if NULL + ggml_opt_result_t result_eval, // result to increment during evaluation, ignored if NULL + int64_t idata_split, // data index at which to split training and evaluation + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval); + + // callback that prints a progress bar on stderr + GGML_API void ggml_opt_epoch_callback_progress_bar( + bool train, + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, + int64_t ibatch, + int64_t ibatch_max, + int64_t t_start_us); + + // fit model defined by inputs and outputs to dataset + GGML_API void ggml_opt_fit( + ggml_backend_sched_t backend_sched, // backend scheduler for constructing the compute graphs + struct ggml_context * ctx_compute, // context with temporarily allocated tensors to calculate the outputs + struct ggml_tensor * inputs, // input tensor with shape [ne_datapoint, ndata_batch] + struct ggml_tensor * outputs, // output tensor, must have shape [ne_label, ndata_batch] if labels are used + ggml_opt_dataset_t dataset, // dataset with data and optionally also labels + enum ggml_opt_loss_type loss_type, // loss to minimize + enum ggml_opt_optimizer_type optimizer, // sgd or adamw + ggml_opt_get_optimizer_params get_opt_pars, // callback to get optimizer params, userdata is pointer to epoch (of type int64_t) + int64_t nepoch, // how many times the dataset should be iterated over + int64_t nbatch_logical, // datapoints optimizer step, must be a multiple of ndata_batch in inputs/outputs + float val_split, // fraction of the dataset to use for validation, must be in [0.0f, 1.0f) + bool silent); // whether or not info prints to stderr should be suppressed + + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml.h new file mode 100644 index 00000000..374934aa --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/ggml.h @@ -0,0 +1,2856 @@ +#pragma once + +// +// GGML Tensor Library +// +// This documentation is still a work in progress. +// If you wish some specific topics to be covered, feel free to drop a comment: +// +// https://github.com/ggml-org/whisper.cpp/issues/40 +// +// ## Overview +// +// This library implements: +// +// - a set of tensor operations +// - automatic differentiation +// - basic optimization algorithms +// +// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes, +// but is not limited to, the following: +// +// - linear regression +// - support vector machines +// - neural networks +// +// The library allows the user to define a certain function using the available tensor operations. This function +// definition is represented internally via a computation graph. Each tensor operation in the function definition +// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the +// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized +// using one of the available optimization algorithms. +// +// For example, here we define the function: f(x) = a*x^2 + b +// +// { +// struct ggml_init_params params = { +// .mem_size = 16*1024*1024, +// .mem_buffer = NULL, +// }; +// +// // memory allocation happens here +// struct ggml_context * ctx = ggml_init(params); +// +// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// +// ggml_set_param(ctx, x); // x is an input variable +// +// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * x2 = ggml_mul(ctx, x, x); +// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b); +// +// ... +// } +// +// Notice that the function definition above does not involve any actual computation. The computation is performed only +// when the user explicitly requests it. For example, to compute the function's value at x = 2.0: +// +// { +// ... +// +// struct ggml_cgraph * gf = ggml_new_graph(ctx); +// ggml_build_forward_expand(gf, f); +// +// // set the input variable and parameter values +// ggml_set_f32(x, 2.0f); +// ggml_set_f32(a, 3.0f); +// ggml_set_f32(b, 4.0f); +// +// ggml_graph_compute_with_ctx(ctx, &gf, n_threads); +// +// printf("f = %f\n", ggml_get_f32_1d(f, 0)); +// +// ... +// } +// +// The actual computation is performed in the ggml_graph_compute() function. +// +// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the +// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know +// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory +// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was +// actually needed. +// +// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic +// differentiation and optimization algorithms. +// +// The described approach allows to define the function graph once and then compute its forward or backward graphs +// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way +// the user can avoid the memory allocation overhead at runtime. +// +// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class +// citizens, but in theory the library can be extended to support FP8 and integer data types. +// +// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary +// and binary operations. Most of the available operations fall into one of these two categories. With time, it became +// clear that the library needs to support more complex operations. The way to support these operations is not clear +// yet, but a few examples are demonstrated in the following operations: +// +// - ggml_permute() +// - ggml_conv_1d_1s() +// - ggml_conv_1d_2s() +// +// For each tensor operator, the library implements a forward and backward computation function. The forward function +// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the +// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a +// calculus class, or watch the following video: +// +// What is Automatic Differentiation? +// https://www.youtube.com/watch?v=wG_nF1awSSY +// +// +// ## Tensor data (struct ggml_tensor) +// +// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of +// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains +// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example: +// +// { +// struct ggml_tensor * c = ggml_add(ctx, a, b); +// +// assert(c->src[0] == a); +// assert(c->src[1] == b); +// } +// +// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the +// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows +// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and +// permutation. All tensor operations have to take the stride into account and not assume that the tensor is +// contiguous in memory. +// +// The data of the tensor is accessed via the "data" pointer. For example: +// +// { +// const int nx = 2; +// const int ny = 3; +// +// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny); +// +// for (int y = 0; y < ny; y++) { +// for (int x = 0; x < nx; x++) { +// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y; +// } +// } +// +// ... +// } +// +// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used. +// +// ## The matrix multiplication operator (ggml_mul_mat) +// +// TODO +// +// +// ## Multi-threading +// +// TODO +// +// +// ## Overview of ggml.c +// +// TODO +// +// +// ## SIMD optimizations +// +// TODO +// +// +// ## Debugging ggml +// +// TODO +// +// + +#ifdef GGML_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BUILD +# define GGML_API __declspec(dllexport) extern +# else +# define GGML_API __declspec(dllimport) extern +# endif +# else +# define GGML_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_API extern +#endif + +// TODO: support for clang +#ifdef __GNUC__ +# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define GGML_DEPRECATED(func, hint) func +#endif + +#ifndef __GNUC__ +# define GGML_ATTRIBUTE_FORMAT(...) +#elif defined(__MINGW32__) && !defined(__clang__) +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__))) +#else +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__))) +#endif + +#if defined(_WIN32) && !defined(_WIN32_WINNT) +# define _WIN32_WINNT 0x0A00 +#endif + +#include +#include +#include +#include + +#define GGML_FILE_MAGIC 0x67676d6c // "ggml" +#define GGML_FILE_VERSION 2 + +#define GGML_QNT_VERSION 2 // bump this on quantization format changes +#define GGML_QNT_VERSION_FACTOR 1000 // do not change this + +#define GGML_MAX_DIMS 4 +#define GGML_MAX_PARAMS 2048 +#define GGML_MAX_SRC 10 +#define GGML_MAX_N_THREADS 512 +#define GGML_MAX_OP_PARAMS 64 + +#ifndef GGML_MAX_NAME +# define GGML_MAX_NAME 64 +#endif + +#define GGML_DEFAULT_N_THREADS 4 +#define GGML_DEFAULT_GRAPH_SIZE 2048 + +#if UINTPTR_MAX == 0xFFFFFFFF + #define GGML_MEM_ALIGN 4 +#elif defined(__EMSCRIPTEN__) +// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm. +// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.) +// ref: https://github.com/ggml-org/llama.cpp/pull/18628 + #define GGML_MEM_ALIGN 8 +#else + #define GGML_MEM_ALIGN 16 +#endif + +#define GGML_EXIT_SUCCESS 0 +#define GGML_EXIT_ABORTED 1 + +// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726 +#define GGML_ROPE_TYPE_NORMAL 0 +#define GGML_ROPE_TYPE_NEOX 2 +#define GGML_ROPE_TYPE_MROPE 8 +#define GGML_ROPE_TYPE_VISION 24 +#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000 + +#define GGML_MROPE_SECTIONS 4 + +#define GGML_UNUSED(x) (void)(x) +#ifdef __CUDACC__ +template +__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {} +#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__) +#else +#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0) +#endif // __CUDACC__ + +#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) + +#ifndef NDEBUG +# define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0) +#elif defined(__GNUC__) +# define GGML_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) +# define GGML_UNREACHABLE() __assume(0) +#else +# define GGML_UNREACHABLE() ((void) 0) +#endif + +#ifdef __cplusplus +# define GGML_NORETURN [[noreturn]] +#elif defined(_MSC_VER) +# define GGML_NORETURN __declspec(noreturn) +#else +# define GGML_NORETURN _Noreturn +#endif + +#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__) +#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x) + +// used to copy the number of elements and stride in bytes of tensors into local variables. +// main purpose is to reduce code duplication and improve readability. +// +// example: +// +// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); +// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb); +// +#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \ + const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \ + GGML_UNUSED(prefix##0); +#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \ + const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \ + GGML_UNUSED(prefix##1); +#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \ + const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \ + GGML_UNUSED(prefix##2); +#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \ + const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \ + GGML_UNUSED(prefix##3); + +#define GGML_TENSOR_UNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_TERNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \ + GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS01 \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) + +#ifdef __cplusplus +extern "C" { +#endif + + // Function type used in fatal error callbacks + typedef void (*ggml_abort_callback_t)(const char * error_message); + + // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout) + // Returns the old callback for chaining + GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback); + + GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4) + GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...); + + enum ggml_status { + GGML_STATUS_ALLOC_FAILED = -2, + GGML_STATUS_FAILED = -1, + GGML_STATUS_SUCCESS = 0, + GGML_STATUS_ABORTED = 1, + }; + + // get ggml_status name string + GGML_API const char * ggml_status_to_string(enum ggml_status status); + + // ieee 754-2008 half-precision float16 + // todo: make this not an integral type + typedef uint16_t ggml_fp16_t; + GGML_API float ggml_fp16_to_fp32(ggml_fp16_t); + GGML_API ggml_fp16_t ggml_fp32_to_fp16(float); + GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t); + + // google brain half-precision bfloat16 + typedef struct { uint16_t bits; } ggml_bf16_t; + GGML_API ggml_bf16_t ggml_fp32_to_bf16(float); + GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16 + GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t); + GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t); + + struct ggml_object; + struct ggml_context; + struct ggml_cgraph; + + // NOTE: always add types at the end of the enum to keep backward compatibility + enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + // GGML_TYPE_Q4_2 = 4, support has been removed + // GGML_TYPE_Q4_3 = 5, support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, + GGML_TYPE_IQ2_XXS = 16, + GGML_TYPE_IQ2_XS = 17, + GGML_TYPE_IQ3_XXS = 18, + GGML_TYPE_IQ1_S = 19, + GGML_TYPE_IQ4_NL = 20, + GGML_TYPE_IQ3_S = 21, + GGML_TYPE_IQ2_S = 22, + GGML_TYPE_IQ4_XS = 23, + GGML_TYPE_I8 = 24, + GGML_TYPE_I16 = 25, + GGML_TYPE_I32 = 26, + GGML_TYPE_I64 = 27, + GGML_TYPE_F64 = 28, + GGML_TYPE_IQ1_M = 29, + GGML_TYPE_BF16 = 30, + // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // GGML_TYPE_Q4_0_4_8 = 32, + // GGML_TYPE_Q4_0_8_8 = 33, + GGML_TYPE_TQ1_0 = 34, + GGML_TYPE_TQ2_0 = 35, + // GGML_TYPE_IQ4_NL_4_4 = 36, + // GGML_TYPE_IQ4_NL_4_8 = 37, + // GGML_TYPE_IQ4_NL_8_8 = 38, + GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) + GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_COUNT = 42, + }; + + // precision + enum ggml_prec { + GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default + GGML_PREC_F32 = 10, + }; + + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, + }; + + // model file types + enum ggml_ftype { + GGML_FTYPE_UNKNOWN = -1, + GGML_FTYPE_ALL_F32 = 0, + GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors + GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors + GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors + GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors + GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + }; + + // available tensor operations: + enum ggml_op { + GGML_OP_NONE = 0, + + GGML_OP_DUP, + GGML_OP_ADD, + GGML_OP_ADD_ID, + GGML_OP_ADD1, + GGML_OP_ACC, + GGML_OP_SUB, + GGML_OP_MUL, + GGML_OP_DIV, + GGML_OP_SQR, + GGML_OP_SQRT, + GGML_OP_LOG, + GGML_OP_SIN, + GGML_OP_COS, + GGML_OP_SUM, + GGML_OP_SUM_ROWS, + GGML_OP_CUMSUM, + GGML_OP_MEAN, + GGML_OP_ARGMAX, + GGML_OP_COUNT_EQUAL, + GGML_OP_REPEAT, + GGML_OP_REPEAT_BACK, + GGML_OP_CONCAT, + GGML_OP_SILU_BACK, + GGML_OP_NORM, // normalize + GGML_OP_RMS_NORM, + GGML_OP_RMS_NORM_BACK, + GGML_OP_GROUP_NORM, + GGML_OP_L2_NORM, + + GGML_OP_MUL_MAT, + GGML_OP_MUL_MAT_ID, + GGML_OP_OUT_PROD, + + GGML_OP_SCALE, + GGML_OP_SET, + GGML_OP_CPY, + GGML_OP_CONT, + GGML_OP_RESHAPE, + GGML_OP_VIEW, + GGML_OP_PERMUTE, + GGML_OP_TRANSPOSE, + GGML_OP_GET_ROWS, + GGML_OP_GET_ROWS_BACK, + GGML_OP_SET_ROWS, + GGML_OP_DIAG, + GGML_OP_DIAG_MASK_INF, + GGML_OP_DIAG_MASK_ZERO, + GGML_OP_SOFT_MAX, + GGML_OP_SOFT_MAX_BACK, + GGML_OP_ROPE, + GGML_OP_ROPE_BACK, + GGML_OP_CLAMP, + GGML_OP_CONV_TRANSPOSE_1D, + GGML_OP_IM2COL, + GGML_OP_IM2COL_BACK, + GGML_OP_IM2COL_3D, + GGML_OP_COL2IM_1D, + GGML_OP_CONV_2D, + GGML_OP_CONV_3D, + GGML_OP_CONV_2D_DW, + GGML_OP_CONV_TRANSPOSE_2D, + GGML_OP_POOL_1D, + GGML_OP_POOL_2D, + GGML_OP_POOL_2D_BACK, + GGML_OP_UPSCALE, + GGML_OP_PAD, + GGML_OP_PAD_REFLECT_1D, + GGML_OP_ROLL, + GGML_OP_ARANGE, + GGML_OP_TIMESTEP_EMBEDDING, + GGML_OP_ARGSORT, + GGML_OP_TOP_K, + GGML_OP_LEAKY_RELU, + GGML_OP_TRI, + GGML_OP_FILL, + + GGML_OP_FLASH_ATTN_EXT, + GGML_OP_FLASH_ATTN_BACK, + GGML_OP_SSM_CONV, + GGML_OP_SSM_SCAN, + GGML_OP_WIN_PART, + GGML_OP_WIN_UNPART, + GGML_OP_GET_REL_POS, + GGML_OP_ADD_REL_POS, + GGML_OP_RWKV_WKV6, + GGML_OP_GATED_LINEAR_ATTN, + GGML_OP_RWKV_WKV7, + GGML_OP_SOLVE_TRI, + GGML_OP_GATED_DELTA_NET, + + GGML_OP_UNARY, + + GGML_OP_MAP_CUSTOM1, + GGML_OP_MAP_CUSTOM2, + GGML_OP_MAP_CUSTOM3, + + GGML_OP_CUSTOM, + + GGML_OP_CROSS_ENTROPY_LOSS, + GGML_OP_CROSS_ENTROPY_LOSS_BACK, + GGML_OP_OPT_STEP_ADAMW, + GGML_OP_OPT_STEP_SGD, + + GGML_OP_GLU, + + GGML_OP_COUNT, + }; + + enum ggml_unary_op { + GGML_UNARY_OP_ABS, + GGML_UNARY_OP_SGN, + GGML_UNARY_OP_NEG, + GGML_UNARY_OP_STEP, + GGML_UNARY_OP_TANH, + GGML_UNARY_OP_ELU, + GGML_UNARY_OP_RELU, + GGML_UNARY_OP_SIGMOID, + GGML_UNARY_OP_GELU, + GGML_UNARY_OP_GELU_QUICK, + GGML_UNARY_OP_SILU, + GGML_UNARY_OP_HARDSWISH, + GGML_UNARY_OP_HARDSIGMOID, + GGML_UNARY_OP_EXP, + GGML_UNARY_OP_EXPM1, + GGML_UNARY_OP_SOFTPLUS, + GGML_UNARY_OP_GELU_ERF, + GGML_UNARY_OP_XIELU, + GGML_UNARY_OP_FLOOR, + GGML_UNARY_OP_CEIL, + GGML_UNARY_OP_ROUND, + GGML_UNARY_OP_TRUNC, + + GGML_UNARY_OP_COUNT, + }; + + enum ggml_glu_op { + GGML_GLU_OP_REGLU, + GGML_GLU_OP_GEGLU, + GGML_GLU_OP_SWIGLU, + GGML_GLU_OP_SWIGLU_OAI, + GGML_GLU_OP_GEGLU_ERF, + GGML_GLU_OP_GEGLU_QUICK, + + GGML_GLU_OP_COUNT, + }; + + enum ggml_object_type { + GGML_OBJECT_TYPE_TENSOR, + GGML_OBJECT_TYPE_GRAPH, + GGML_OBJECT_TYPE_WORK_BUFFER + }; + + enum ggml_log_level { + GGML_LOG_LEVEL_NONE = 0, + GGML_LOG_LEVEL_DEBUG = 1, + GGML_LOG_LEVEL_INFO = 2, + GGML_LOG_LEVEL_WARN = 3, + GGML_LOG_LEVEL_ERROR = 4, + GGML_LOG_LEVEL_CONT = 5, // continue previous log + }; + + // this tensor... + enum ggml_tensor_flag { + GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph + GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph + GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters + GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up) + GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed + }; + + enum ggml_tri_type { + GGML_TRI_TYPE_UPPER_DIAG = 0, + GGML_TRI_TYPE_UPPER = 1, + GGML_TRI_TYPE_LOWER_DIAG = 2, + GGML_TRI_TYPE_LOWER = 3 + }; + + struct ggml_init_params { + // memory pool + size_t mem_size; // bytes + void * mem_buffer; // if NULL, memory will be allocated internally + bool no_alloc; // don't allocate memory for the tensor data + }; + + // n-dimensional tensor + struct ggml_tensor { + enum ggml_type type; + + struct ggml_backend_buffer * buffer; + + int64_t ne[GGML_MAX_DIMS]; // number of elements + size_t nb[GGML_MAX_DIMS]; // stride in bytes: + // nb[0] = ggml_type_size(type) + // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding + // nb[i] = nb[i-1] * ne[i-1] + + // compute data + enum ggml_op op; + + // op params - allocated as int32_t for alignment + int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; + + int32_t flags; + + struct ggml_tensor * src[GGML_MAX_SRC]; + + // source tensor and offset for views + struct ggml_tensor * view_src; + size_t view_offs; + + void * data; + + char name[GGML_MAX_NAME]; + + void * extra; // extra things e.g. for ggml-cuda.cu + + char padding[8]; + }; + + static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); + + // Abort callback + // If not NULL, called before ggml computation + // If it returns true, the computation is aborted + typedef bool (*ggml_abort_callback)(void * data); + + + // + // GUID + // + + // GUID types + typedef uint8_t ggml_guid[16]; + typedef ggml_guid * ggml_guid_t; + + GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b); + + // misc + + GGML_API const char * ggml_version(void); + GGML_API const char * ggml_commit(void); + + GGML_API void ggml_time_init(void); // call this once at the beginning of the program + GGML_API int64_t ggml_time_ms(void); + GGML_API int64_t ggml_time_us(void); + GGML_API int64_t ggml_cycles(void); + GGML_API int64_t ggml_cycles_per_ms(void); + + // accepts a UTF-8 path, even on Windows + GGML_API FILE * ggml_fopen(const char * fname, const char * mode); + + GGML_API void ggml_print_object (const struct ggml_object * obj); + GGML_API void ggml_print_objects(const struct ggml_context * ctx); + + GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor); + GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN + + GGML_API int64_t ggml_blck_size(enum ggml_type type); + GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block + GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row + + GGML_DEPRECATED( + GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float + "use ggml_row_size() instead"); + + GGML_API const char * ggml_type_name(enum ggml_type type); + GGML_API const char * ggml_op_name (enum ggml_op op); + GGML_API const char * ggml_op_symbol(enum ggml_op op); + + GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op); + GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op); + GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name + + GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor); + + GGML_API bool ggml_is_quantized(enum ggml_type type); + + // TODO: temporary until model loading of ggml examples is refactored + GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype); + + GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor); + GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_view (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor); + GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars + + // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation) + GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous() + GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1 + GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2 + + // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok) + GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor); + + // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN + GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor); + + // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements + GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor); + + GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1); + GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + // use this to compute the memory overhead of a tensor + GGML_API size_t ggml_tensor_overhead(void); + + GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes); + + // main + + GGML_API struct ggml_context * ggml_init (struct ggml_init_params params); + GGML_API void ggml_reset(struct ggml_context * ctx); + GGML_API void ggml_free (struct ggml_context * ctx); + + GGML_API size_t ggml_used_mem(const struct ggml_context * ctx); + + GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx); + GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc); + + GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx); + GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx); + GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx); + + GGML_API struct ggml_tensor * ggml_new_tensor( + struct ggml_context * ctx, + enum ggml_type type, + int n_dims, + const int64_t *ne); + + GGML_API struct ggml_tensor * ggml_new_tensor_1d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_new_tensor_2d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_new_tensor_3d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_new_tensor_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes); + + GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src); + GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src); + + // Context tensor enumeration and lookup + GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx); + GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name); + + // Converts a flat index into coordinates + GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3); + + GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor); + GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor); + + GGML_API void * ggml_get_data (const struct ggml_tensor * tensor); + GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor); + + GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name); + GGML_ATTRIBUTE_FORMAT(2, 3) + GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...); + + // Tensor flags + GGML_API void ggml_set_input(struct ggml_tensor * tensor); + GGML_API void ggml_set_output(struct ggml_tensor * tensor); + GGML_API void ggml_set_param(struct ggml_tensor * tensor); + GGML_API void ggml_set_loss(struct ggml_tensor * tensor); + + // + // operations on tensors with backpropagation + // + + GGML_API struct ggml_tensor * ggml_dup( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_dup_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_add( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_type type); + + // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]] + GGML_API struct ggml_tensor * ggml_add_id( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add_inplace instead"); + + // dst = a + // view(dst, nb1, nb2, nb3, offset) += b + // return dst + GGML_API struct ggml_tensor * ggml_acc( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_acc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_sub( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sub_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sqr( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqr_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // return scalar + GGML_API struct ggml_tensor * ggml_sum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d] + GGML_API struct ggml_tensor * ggml_sum_rows( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cumsum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // mean along rows + GGML_API struct ggml_tensor * ggml_mean( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // argmax along rows + GGML_API struct ggml_tensor * ggml_argmax( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // count number of equal elements in a and b + GGML_API struct ggml_tensor * ggml_count_equal( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // if a is the same shape as b, and a is not parameter, return a + // otherwise, return a new tensor: repeat(a) to fit in b + GGML_API struct ggml_tensor * ggml_repeat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // repeat a to the specified shape + GGML_API struct ggml_tensor * ggml_repeat_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // sums repetitions in a into shape of b + GGML_API struct ggml_tensor * ggml_repeat_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride + + // concat a and b along dim + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_concat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int dim); + + GGML_API struct ggml_tensor * ggml_abs( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_abs_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_relu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_leaky_relu( + struct ggml_context * ctx, + struct ggml_tensor * a, float negative_slope, bool inplace); + + GGML_API struct ggml_tensor * ggml_relu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // GELU using erf (error function) when possible + // some backends may fallback to approximation based on Abramowitz and Stegun formula + GGML_API struct ggml_tensor * ggml_gelu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_erf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a - dy + // b - x + GGML_API struct ggml_tensor * ggml_silu_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // hardswish(x) = x * relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardswish( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // hardsigmoid(x) = relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardsigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + /** + * Truncates the fractional part of each element in the tensor (towards zero). + * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0 + * Similar to std::trunc in C/C++. + */ + + GGML_API struct ggml_tensor * ggml_trunc( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_trunc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + + + // xIELU activation function + // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0) + // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions + // that constrain the positive and negative source alpha values respectively + GGML_API struct ggml_tensor * ggml_xielu( + struct ggml_context * ctx, + struct ggml_tensor * a, + float alpha_n, + float alpha_p, + float beta, + float eps); + + // gated linear unit ops + // A: n columns, r rows, + // result is n / 2 columns, r rows, + // expects gate in second half of row, unless swapped is true + GGML_API struct ggml_tensor * ggml_glu( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_glu_op op, + bool swapped); + + GGML_API struct ggml_tensor * ggml_reglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_reglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // A: n columns, r rows, + // B: n columns, r rows, + GGML_API struct ggml_tensor * ggml_glu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_glu_op op); + + GGML_API struct ggml_tensor * ggml_reglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_erf_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_quick_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_oai( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float alpha, + float limit); + + // normalize along rows + GGML_API struct ggml_tensor * ggml_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // group normalize along ne0*ne1*n_groups + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_group_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + GGML_API struct ggml_tensor * ggml_group_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + // l2 normalize along rows + // used in rwkv v7 + GGML_API struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // a - x + // b - dy + GGML_API struct ggml_tensor * ggml_rms_norm_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float eps); + + // A: k columns, n rows => [ne03, ne02, n, k] + // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] + // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] + GGML_API struct ggml_tensor * ggml_mul_mat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // change the precision of a matrix multiplication + // set to GGML_PREC_F32 for higher precision (useful for phi-2) + GGML_API void ggml_mul_mat_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); + + // indirect matrix multiplication + GGML_API struct ggml_tensor * ggml_mul_mat_id( + struct ggml_context * ctx, + struct ggml_tensor * as, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + // A: m columns, n rows, + // B: p columns, n rows, + // result is m columns, p rows + GGML_API struct ggml_tensor * ggml_out_prod( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // + // operations on tensors without backpropagation + // + + GGML_API struct ggml_tensor * ggml_scale( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_scale_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // x = s * a + b + GGML_API struct ggml_tensor * ggml_scale_bias( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + GGML_API struct ggml_tensor * ggml_scale_bias_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_2d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // a -> b, return view(b) + GGML_API struct ggml_tensor * ggml_cpy( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // note: casting from f32 to i32 will discard the fractional part + GGML_API struct ggml_tensor * ggml_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_type type); + + // make contiguous + GGML_API struct ggml_tensor * ggml_cont( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // make contiguous, with new shape + GGML_API struct ggml_tensor * ggml_cont_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_cont_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_cont_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_cont_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // return view(a), b specifies the new shape + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_reshape_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_reshape_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // offset in bytes + GGML_API struct ggml_tensor * ggml_view_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + size_t nb1, // row stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_permute( + struct ggml_context * ctx, + struct ggml_tensor * a, + int axis0, + int axis1, + int axis2, + int axis3); + + // alias for ggml_permute(ctx, a, 1, 0, 2, 3) + GGML_API struct ggml_tensor * ggml_transpose( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // supports 4D a: + // a [n_embd, ne1, ne2, ne3] + // b I32 [n_rows, ne2, ne3, 1] + // + // return [n_embd, n_rows, ne2, ne3] + GGML_API struct ggml_tensor * ggml_get_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // data + struct ggml_tensor * b); // row indices + + GGML_API struct ggml_tensor * ggml_get_rows_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_get_rows result + struct ggml_tensor * b, // row indices + struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape + + // a TD [n_embd, ne1, ne2, ne3] + // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3 + // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1) + // + // undefined behavior if destination rows overlap + // + // broadcast: + // ne2 % ne11 == 0 + // ne3 % ne12 == 0 + // + // return view(a) + GGML_API struct ggml_tensor * ggml_set_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // destination + struct ggml_tensor * b, // source + struct ggml_tensor * c); // row indices + + GGML_API struct ggml_tensor * ggml_diag( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // set elements above the diagonal to -INF + GGML_API struct ggml_tensor * ggml_diag_mask_inf( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // set elements above the diagonal to 0 + GGML_API struct ggml_tensor * ggml_diag_mask_zero( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + GGML_API struct ggml_tensor * ggml_soft_max( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a [ne0, ne01, ne02, ne03] + // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional + // + // broadcast: + // ne02 % ne12 == 0 + // ne03 % ne13 == 0 + // + // fused soft_max(a*scale + mask*(ALiBi slope)) + // max_bias = 0.0f for no ALiBi + GGML_API struct ggml_tensor * ggml_soft_max_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API void ggml_soft_max_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // rotary position embedding + // if (mode & 1) - skip n_past elements (NOT SUPPORTED) + // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style + // + // b is an int32 vector with size a->ne[2], it contains the positions + GGML_API struct ggml_tensor * ggml_rope( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // RoPE operations with extended options + // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token] + // b is an int32 vector with size n_token + // c is freq factors (e.g. phi3-128k), (optional) + // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi + // + // pseudo-code for computing theta: + // for i in [0, n_dims/2): + // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims); + // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c + // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here + // + // other params are used by YaRN RoPE scaling, these default values will disable YaRN: + // freq_scale = 1.0f + // ext_factor = 0.0f + // attn_factor = 1.0f + // beta_fast = 0.0f + // beta_slow = 0.0f + // + // example: + // (marking: c = cos, s = sin, 0 = unrotated) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000] + // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs] + // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000] + // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss] + GGML_API struct ggml_tensor * ggml_rope_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // multi-dimensional RoPE, for Qwen-VL and similar vision models + // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX + // sections specify how many dimensions to rotate in each section: + // section length is equivalent to number of cos/sin pairs, NOT the number of dims + // (i.e. sum of 4 sections are expected to be n_dims/2) + // last sections can be 0, means ignored + // all other options are identical to ggml_rope_ext + // + // important note: + // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION + // if you need normal ordering, there are 2 methods: + // (1) split the tensor manually using ggml_view + // (2) permute the weight upon conversion + // - for VISION, n_dims must be head_size/2 + // + // example M-RoPE: + // given sections = [t=4, y=2, x=2, 0] + // given a single head with size = 18 --> [000000000000000000] + // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering) + // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering) + // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section + // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section + // + // example vision RoPE: + // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx] + // other values of n_dims are untested and is undefined behavior + // note: unlike MROPE, the theta for each dim is computed differently for each section + // in other words, idx used for theta: [0123] for y section, then [0123] for x section + GGML_API struct ggml_tensor * ggml_rope_multi( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext_inplace instead"); + + // compute correction dims for YaRN RoPE scaling + GGML_API void ggml_rope_yarn_corr_dims( + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]); + + // rotary position embedding backward, i.e compute dx from dy + // a - dy + GGML_API struct ggml_tensor * ggml_rope_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_rope result + struct ggml_tensor * b, // positions + struct ggml_tensor * c, // freq factors + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[4], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + + // clamp + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_clamp( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + + // im2col + // converts data into a format that effectively results in a convolution when combined with matrix multiplication + GGML_API struct ggml_tensor * ggml_im2col( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D, + enum ggml_type dst_type); + + GGML_API struct ggml_tensor * ggml_im2col_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // gradient of im2col output + int64_t * ne, // shape of im2col input + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D); + + // col2im_1d: scatter-add GEMM columns back to 1D signal + // a: [K*OC, T_in] (columns from matmul, K = a->ne[0]/OC) + // result: [T_out, OC] where T_out = (T_in - 1)*s0 + K - 2*p0 + GGML_API struct ggml_tensor * ggml_col2im_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // columns [K*OC, T_in] + int s0, // stride + int oc, // output channels + int p0); // padding to crop from both sides + + GGML_API struct ggml_tensor * ggml_conv_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + // conv_1d with padding = half + // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d) + GGML_API struct ggml_tensor* ggml_conv_1d_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s, // stride + int d); // dilation + + // depthwise + // TODO: this is very likely wrong for some cases! - needs more testing + GGML_API struct ggml_tensor * ggml_conv_1d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_transpose_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_im2col_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2, // dilation depth + enum ggml_type dst_type); + + // a: [OC*IC, KD, KH, KW] + // b: [N*IC, ID, IH, IW] + // result: [N*OC, OD, OH, OW] + GGML_API struct ggml_tensor * ggml_conv_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2 // dilation depth + ); + + // kernel size is a->ne[0] x a->ne[1] + // stride is equal to kernel size + // padding is zero + // example: + // a: 16 16 3 768 + // b: 1024 1024 3 1 + // res: 64 64 768 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // kernel size is a->ne[0] x a->ne[1] + // stride is 1 + // padding is half + // example: + // a: 3 3 256 256 + // b: 64 64 256 1 + // res: 64 64 256 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // depthwise (via im2col and mul_mat) + GGML_API struct ggml_tensor * ggml_conv_2d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + // Depthwise 2D convolution + // may be faster than ggml_conv_2d_dw, but not available in all backends + // a: KW KH 1 C convolution kernel + // b: W H C N input data + // res: W_out H_out C N + GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride0, + int stride1, + int pad0, + int pad1, + int dilation0, + int dilation1); + + GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride); + + GGML_API struct ggml_tensor * ggml_conv_2d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC] + struct ggml_tensor * b, // input data [W, H, C, N] + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_conv_3d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC] + struct ggml_tensor * b, // input [W, H, D, C * N] + int s0, // stride + int s1, + int s2, + int p0, // padding + int p1, + int p2, + int d0, // dilation + int d1, + int d2, + int n_channels, + int n_batch, + int n_channels_out); + + enum ggml_op_pool { + GGML_OP_POOL_MAX, + GGML_OP_POOL_AVG, + GGML_OP_POOL_COUNT, + }; + + GGML_API struct ggml_tensor * ggml_pool_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, // kernel size + int s0, // stride + int p0); // padding + + // the result will have 2*p0 padding for the first dimension + // and 2*p1 padding for the second dimension + GGML_API struct ggml_tensor * ggml_pool_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + GGML_API struct ggml_tensor * ggml_pool_2d_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * af, // "a"/input used in forward pass + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + enum ggml_scale_mode { + GGML_SCALE_MODE_NEAREST = 0, + GGML_SCALE_MODE_BILINEAR = 1, + GGML_SCALE_MODE_BICUBIC = 2, + + GGML_SCALE_MODE_COUNT + }; + + enum ggml_scale_flag { + GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8), + GGML_SCALE_FLAG_ANTIALIAS = (1 << 9), + }; + + // interpolate + // multiplies ne0 and ne1 by scale factor + GGML_API struct ggml_tensor * ggml_upscale( + struct ggml_context * ctx, + struct ggml_tensor * a, + int scale_factor, + enum ggml_scale_mode mode); + + // interpolate + // interpolate scale to specified dimensions + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int ne0, + int ne1, + int ne2, + int ne3, + enum ggml_scale_mode mode), + "use ggml_interpolate instead"); + + // Up- or downsamples the input to the specified size. + // 2D scale modes (eg. bilinear) are applied to the first two dimensions. + GGML_API struct ggml_tensor * ggml_interpolate( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...] + + // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0] + GGML_API struct ggml_tensor * ggml_pad( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + GGML_API struct ggml_tensor * ggml_pad_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3 + ); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_ext_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3); + + // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c] + GGML_API struct ggml_tensor * ggml_pad_reflect_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1); + + // Move tensor elements by an offset given for each dimension. Elements that + // are shifted beyond the last position are wrapped around to the beginning. + GGML_API struct ggml_tensor * ggml_roll( + struct ggml_context * ctx, + struct ggml_tensor * a, + int shift0, + int shift1, + int shift2, + int shift3); + + // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing + // zeroes everywhere outside the masked area + GGML_API struct ggml_tensor * ggml_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_tri_type type); + + // Fill tensor a with constant c + GGML_API struct ggml_tensor * ggml_fill( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + GGML_API struct ggml_tensor * ggml_fill_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151 + // timesteps: [N,] + // return: [N, dim] + GGML_API struct ggml_tensor * ggml_timestep_embedding( + struct ggml_context * ctx, + struct ggml_tensor * timesteps, + int dim, + int max_period); + + // sort rows + enum ggml_sort_order { + GGML_SORT_ORDER_ASC, + GGML_SORT_ORDER_DESC, + }; + + GGML_API struct ggml_tensor * ggml_argsort( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_sort_order order); + + // similar to ggml_top_k but implemented as `argsort` + `view` + GGML_API struct ggml_tensor * ggml_argsort_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + // top k elements per row + // note: the resulting top k indices are in no particular order + GGML_API struct ggml_tensor * ggml_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + GGML_API struct ggml_tensor * ggml_arange( + struct ggml_context * ctx, + float start, + float stop, + float step); + + // q: [n_embd_k, n_batch, n_head, ne3 ] + // k: [n_embd_k, n_kv, n_head_kv, ne3 ] + // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !! + // mask: [n_kv, n_batch, ne32, ne33] + // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !! + // + // broadcast: + // n_head % n_head_kv == 0 + // n_head % ne32 == 0 + // ne3 % ne33 == 0 + // + GGML_API struct ggml_tensor * ggml_flash_attn_ext( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + float scale, + float max_bias, + float logit_softcap); + + GGML_API void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( + const struct ggml_tensor * a); + + GGML_API void ggml_flash_attn_ext_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + // TODO: needs to be adapted to ggml_flash_attn_ext + GGML_API struct ggml_tensor * ggml_flash_attn_back( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * d, + bool masked); + + GGML_API struct ggml_tensor * ggml_ssm_conv( + struct ggml_context * ctx, + struct ggml_tensor * sx, + struct ggml_tensor * c); + + GGML_API struct ggml_tensor * ggml_ssm_scan( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * dt, + struct ggml_tensor * A, + struct ggml_tensor * B, + struct ggml_tensor * C, + struct ggml_tensor * ids); + + // partition into non-overlapping windows with padding if needed + // example: + // a: 768 64 64 1 + // w: 14 + // res: 768 14 14 25 + // used in sam + GGML_API struct ggml_tensor * ggml_win_part( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w); + + // reverse of ggml_win_part + // used in sam + GGML_API struct ggml_tensor * ggml_win_unpart( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w0, + int h0, + int w); + + GGML_API struct ggml_tensor * ggml_unary( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + GGML_API struct ggml_tensor * ggml_unary_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + // used in sam + GGML_API struct ggml_tensor * ggml_get_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + int qh, + int kh); + + // used in sam + GGML_API struct ggml_tensor * ggml_add_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv6( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * r, + struct ggml_tensor * tf, + struct ggml_tensor * td, + struct ggml_tensor * state); + + GGML_API struct ggml_tensor * ggml_gated_linear_attn( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * q, + struct ggml_tensor * g, + struct ggml_tensor * state, + float scale); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state); + + /* Solves a specific equation of the form Ax=B, where A is a triangular matrix + * without zeroes on the diagonal (i.e. invertible). + * B can have any number of columns, but must have the same number of rows as A + * If A is [n, n] and B is [n, m], then the result will be [n, m] as well + * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases + * where n > 100 sparingly, pre-chunk if necessary. + * + * If left = false, solves xA=B instead + * If lower = false, assumes upper triangular instead + * If uni = true, assumes diagonal of A to be all ones (will override actual values) + * + * TODO: currently only lower, right, non-unitriangular variant is implemented + */ + GGML_API struct ggml_tensor * ggml_solve_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + bool left, + bool lower, + bool uni); + + // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST] + // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306 + // + // state is a 3D tensor of shape (S_v*S_v*H, K, n_seqs): + // K == 1: output carries the final state only. + // K > 1: output carries K snapshot slots; the kernel writes the last min(n_tokens, K) + // per-token snapshots into the trailing slots + GGML_API struct ggml_tensor * ggml_gated_delta_net( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * g, + struct ggml_tensor * beta, + struct ggml_tensor * state); + + // custom operators + + typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata); + typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata); + typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata); + +#define GGML_N_TASKS_MAX (-1) + // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks + + GGML_API struct ggml_tensor * ggml_map_custom1( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + // loss function + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b); // labels + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b, // labels + struct ggml_tensor * c); // gradients of cross_entropy_loss result + + // AdamW optimizer step + // Paper: https://arxiv.org/pdf/1711.05101v3.pdf + // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html + GGML_API struct ggml_tensor * ggml_opt_step_adamw( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * m, + struct ggml_tensor * v, + struct ggml_tensor * adamw_params); // parameters such as the learning rate + + // stochastic gradient descent step (with weight decay) + GGML_API struct ggml_tensor * ggml_opt_step_sgd( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * sgd_params); // alpha, weight decay + + // build forward multiple tensors and select one of them for computing + // this is useful for creating graphs that have constant topology but compute different things based on the input + // ref: https://github.com/ggml-org/llama.cpp/pull/18550 + // + // nodes: + // | - build forward into the graph but do not compute + // c - build forward into the graph and compute + // + // | | ... c ... | + // | | ... c ... | + // | | ... c ... | + // [0 1 ... idx ... n-1] <-- ggml_build_forward_select(..., n, idx) + // c + // c + // + // example: + // struct ggml_tensor * curs[3]; + // + // curs[0] = compute0(...); + // curs[1] = compute1(...); + // curs[2] = compute2(...); + // + // int idx = select_branch(some_input); + // + // struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx); + // + GGML_API struct ggml_tensor * ggml_build_forward_select( + struct ggml_cgraph * cgraph, + struct ggml_tensor ** tensors, + int n_tensors, + int idx); + + GGML_API void ggml_build_forward_expand( + struct ggml_cgraph * cgraph, + struct ggml_tensor * tensor); + + GGML_API void ggml_build_backward_expand( + struct ggml_context * ctx, // context for gradient computation + struct ggml_cgraph * cgraph, + struct ggml_tensor ** grad_accs); + + // graph allocation in a context + GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false + GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads); + GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads); + GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst); + GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1 + GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph); + + GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph); + GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i] + GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph); + GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph); + + GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor); + + GGML_API size_t ggml_graph_overhead(void); + GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads); + + GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name); + GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + + // print info and performance information for the graph + GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph); + + // dump the graph into a file using the dot format + GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename); + + // TODO these functions were sandwiched in the old optimization interface, is there a better place for them? + typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data); + GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data); + + GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor); + + // + // quantization + // + + // - ggml_quantize_init can be called multiple times with the same type + // it will only initialize the quantization tables for the first call or after ggml_quantize_free + // automatically called by ggml_quantize_chunk for convenience + // + // - ggml_quantize_free will free any memory allocated by ggml_quantize_init + // call this at the end of the program to avoid memory leaks + // + // note: these are thread-safe + // + GGML_API void ggml_quantize_init(enum ggml_type type); + GGML_API void ggml_quantize_free(void); + + // some quantization type cannot be used without an importance matrix + GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type); + + // calls ggml_quantize_init internally (i.e. can allocate memory) + GGML_API size_t ggml_quantize_chunk( + enum ggml_type type, + const float * src, + void * dst, + int64_t start, + int64_t nrows, + int64_t n_per_row, + const float * imatrix); + +#ifdef __cplusplus + // restrict not standard in C++ +# if defined(__GNUC__) +# define GGML_RESTRICT __restrict__ +# elif defined(__clang__) +# define GGML_RESTRICT __restrict +# elif defined(_MSC_VER) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT +# endif +#else +# if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT restrict +# endif +#endif + typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + + struct ggml_type_traits { + const char * type_name; + int64_t blck_size; + int64_t blck_size_interleave; // interleave elements in blocks + size_t type_size; + bool is_quantized; + ggml_to_float_t to_float; + ggml_from_float_t from_float_ref; + }; + + GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type); + + // ggml threadpool + // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend + // the goal should be to create an API that other backends can use move everything to the ggml base + + // scheduling priorities + enum ggml_sched_priority { + GGML_SCHED_PRIO_LOW = -1, + GGML_SCHED_PRIO_NORMAL, + GGML_SCHED_PRIO_MEDIUM, + GGML_SCHED_PRIO_HIGH, + GGML_SCHED_PRIO_REALTIME + }; + + // threadpool params + // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults + struct ggml_threadpool_params { + bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings) + int n_threads; // number of threads + enum ggml_sched_priority prio; // thread priority + uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling) + bool strict_cpu; // strict cpu placement + bool paused; // start in paused state + }; + + struct ggml_threadpool; // forward declaration, see ggml.c + + typedef struct ggml_threadpool * ggml_threadpool_t; + + GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads); + GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads); + GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/gguf.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/gguf.h new file mode 100644 index 00000000..67851ba6 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/gguf.h @@ -0,0 +1,210 @@ +// This file contains functionality related to "GGUF" files, the binary file format used by ggml. +// GGUF files have the following structure: +// +// 1. File magic "GGUF" (4 bytes). +// 2. File version (uint32_t). +// 3. Number of ggml tensors in file (int64_t). +// 4. Number of key-value-pairs in file (int64_t). +// 5. For each KV pair: +// 1. The key (string). +// 2. The value type (gguf_type). +// 3a. If the value type is GGUF_TYPE_ARRAY: +// 1. The type of the array (gguf_type). +// 2. The number of elements in the array (uint64_t). +// 3. The binary representation of each element in the array. +// 3b. Otherwise: +// 1. The binary representation of the value. +// 6. For each ggml tensor: +// 1. The tensor name (string). +// 2. The number of dimensions of the tensor (uint32_t). +// 3. For each dimension: +// 1. The size of the tensor in the dimension (int64_t). +// 4. The tensor data type (ggml_type). +// 5. The tensor data offset in the tensor data binary blob (uint64_t). +// 7. The tensor data binary blob (optional, aligned). +// +// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator. +// All enums are stored as int32_t. +// All bool values are stored as int8_t. +// If the special key "general.alignment" (uint32_t) is defined it is used for alignment, +// otherwise GGUF_DEFAULT_ALIGNMENT is used. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" + +#include +#include + +#define GGUF_MAGIC "GGUF" +#define GGUF_VERSION 3 + +#define GGUF_KEY_GENERAL_ALIGNMENT "general.alignment" + +#define GGUF_DEFAULT_ALIGNMENT 32 + +#ifdef __cplusplus +extern "C" { +#endif + + // types that can be stored as GGUF KV data + enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, + GGUF_TYPE_UINT64 = 10, + GGUF_TYPE_INT64 = 11, + GGUF_TYPE_FLOAT64 = 12, + GGUF_TYPE_COUNT, // marks the end of the enum + }; + + struct gguf_context; + + struct gguf_init_params { + bool no_alloc; + + // if not NULL, create a ggml_context and allocate the tensor data in it + struct ggml_context ** ctx; + }; + + // callback to simulate or wrap a FILE pointer - read up to `len` bytes at `offset` into `output` and return the number of bytes read + typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len); + + GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); + + // max_chunk_read is the maximum number of bytes that the GGUF code will read at once from the callback, a value of 0 means no limit + GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params); + + GGML_API void gguf_free(struct gguf_context * ctx); + + GGML_API const char * gguf_type_name(enum gguf_type type); + + GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx); + GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx); + GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); // padded to gguf_get_alignment if and only if the gguf_context contains at least one tensor + + GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx); + GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found + GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id); + + GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id); + GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id); + + // will abort if the wrong type is used for the key + GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id); + GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id); + GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id); + GGML_API size_t gguf_get_arr_n (const struct gguf_context * ctx, int64_t key_id); + + // get raw pointer to the first element of the array with the given key_id + // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference) + GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id); + + // get ith C string from array with given key_id + GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i); + + GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx); + GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found + GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id); + GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id); + + // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist) + GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key); + + // overrides an existing KV pair or adds a new one, the new KV pair is always at the back + GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val); + GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val); + GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val); + GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val); + GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val); + GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val); + GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val); + GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val); + GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val); + GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val); + GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val); + GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val); + + // creates a new array with n elements of the given type and copies the corresponding number of bytes from data + GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n); + + // creates a new array with n strings and copies the corresponding strings from data + GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n); + + // set or add KV pairs from another context + GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src); + + // add tensor to GGUF context, tensor name must be unique + GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor); + + // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated + // in such a way that the tensor data remains as one contiguous block (except for padding) + GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type); + + // assumes that at least gguf_get_tensor_size bytes can be read from data + GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data); + + // writing gguf files can be done in 3 ways: + // + // - write the entire gguf_context to a binary file in a single pass: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ false); + // + // - write only the meta data to a file, then re-open the file and append the tensor data: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ true); + // FILE * f = fopen(fname, "ab"); + // fwrite(f, ...); // write tensor data + // fclose(f); + // + // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data: + // + // FILE * f = fopen(fname, "wb"); + // const size_t size_meta = gguf_get_meta_size(ctx); + // fseek(f, size_meta, SEEK_SET); + // fwrite(f, ...); // write tensor data + // void * data = malloc(size_meta); + // gguf_get_meta_data(ctx, data); + // rewind(f); + // fwrite(data, 1, data, f); + // free(data); + // fclose(f); + // + + // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); + GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); + + // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding + GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx); + + // writes the meta data to pointer "data" + GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/llama.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/llama.h new file mode 100644 index 00000000..27e48067 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/llama.h @@ -0,0 +1,1588 @@ +#ifndef LLAMA_H +#define LLAMA_H + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-backend.h" +#include "ggml-opt.h" +#include "gguf.h" + +#include +#include +#include +#include + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define LLAMA_API __declspec(dllexport) +# else +# define LLAMA_API __declspec(dllimport) +# endif +# else +# define LLAMA_API __attribute__ ((visibility ("default"))) +# endif +#else +# define LLAMA_API +#endif + +#ifdef __GNUC__ +# define DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define DEPRECATED(func, hint) func +#endif + +#define LLAMA_DEFAULT_SEED 0xFFFFFFFF + +#define LLAMA_TOKEN_NULL -1 + +#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' +#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' +#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq' + +#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN +#define LLAMA_SESSION_VERSION 9 + +#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ +#define LLAMA_STATE_SEQ_VERSION 2 + +#ifdef __cplusplus +extern "C" { +#endif + + // + // C interface + // + // TODO: show sample usage + // + + struct llama_vocab; + struct llama_model; + struct llama_context; + struct llama_sampler; + + typedef struct llama_memory_i * llama_memory_t; + + typedef int32_t llama_pos; + typedef int32_t llama_token; + typedef int32_t llama_seq_id; + + enum llama_vocab_type { + LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab + LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback + LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE + LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece + LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram + LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization + LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming + }; + + enum llama_rope_type { + LLAMA_ROPE_TYPE_NONE = -1, + LLAMA_ROPE_TYPE_NORM = 0, + LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, + LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE, + LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE, + LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION, + }; + + enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file + LLAMA_TOKEN_TYPE_UNDEFINED = 0, + LLAMA_TOKEN_TYPE_NORMAL = 1, + LLAMA_TOKEN_TYPE_UNKNOWN = 2, + LLAMA_TOKEN_TYPE_CONTROL = 3, + LLAMA_TOKEN_TYPE_USER_DEFINED = 4, + LLAMA_TOKEN_TYPE_UNUSED = 5, + LLAMA_TOKEN_TYPE_BYTE = 6, + }; + + enum llama_token_attr { + LLAMA_TOKEN_ATTR_UNDEFINED = 0, + LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0, + LLAMA_TOKEN_ATTR_UNUSED = 1 << 1, + LLAMA_TOKEN_ATTR_NORMAL = 1 << 2, + LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL? + LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4, + LLAMA_TOKEN_ATTR_BYTE = 1 << 5, + LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6, + LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7, + LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8, + LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9, + }; + + // model file types + enum llama_ftype { + LLAMA_FTYPE_ALL_F32 = 0, + LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed + // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed + LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors + LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors + //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack + //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack + //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack + LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors + LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors + LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors + LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors + + LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file + }; + + enum llama_rope_scaling_type { + LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, + LLAMA_ROPE_SCALING_TYPE_NONE = 0, + LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, + LLAMA_ROPE_SCALING_TYPE_YARN = 2, + LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, + LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE, + }; + + enum llama_pooling_type { + LLAMA_POOLING_TYPE_UNSPECIFIED = -1, + LLAMA_POOLING_TYPE_NONE = 0, + LLAMA_POOLING_TYPE_MEAN = 1, + LLAMA_POOLING_TYPE_CLS = 2, + LLAMA_POOLING_TYPE_LAST = 3, + LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph + }; + + enum llama_attention_type { + LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1, + LLAMA_ATTENTION_TYPE_CAUSAL = 0, + LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1, + }; + + enum llama_flash_attn_type { + LLAMA_FLASH_ATTN_TYPE_AUTO = -1, + LLAMA_FLASH_ATTN_TYPE_DISABLED = 0, + LLAMA_FLASH_ATTN_TYPE_ENABLED = 1, + }; + + LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); + + enum llama_split_mode { + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_TENSOR = 3, + }; + + enum llama_context_type { + LLAMA_CONTEXT_TYPE_DEFAULT = 0, + LLAMA_CONTEXT_TYPE_MTP = 1, + }; + + // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) + typedef struct llama_token_data { + llama_token id; // token id + float logit; // log-odds of the token + float p; // probability of the token + } llama_token_data; + + typedef struct llama_token_data_array { + // TODO: consider SoA + // NOTE: this pointer can be modified by the samplers + llama_token_data * data; + size_t size; + int64_t selected; // this is the index in the data array (i.e. not the token id) + bool sorted; // note: do not assume the data is sorted - always check this flag + } llama_token_data_array; + + typedef bool (*llama_progress_callback)(float progress, void * user_data); + + // Input data for llama_encode/llama_decode + // A llama_batch object can contain input about one or many sequences + // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens + // + // - token : the token ids of the input (used when embd is NULL) + // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) + // - pos : the positions of the respective token in the sequence + // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode) + // - seq_id : the sequence to which the respective token belongs + // (if set to NULL, the sequence ID will be assumed to be 0) + // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output + // (if set to NULL: + // - if embeddings: all tokens are output + // - if not: only the last token is output + // ) + // + typedef struct llama_batch { + int32_t n_tokens; + + llama_token * token; + float * embd; + llama_pos * pos; + int32_t * n_seq_id; + llama_seq_id ** seq_id; + int8_t * logits; // TODO: rename this to "output" + } llama_batch; + + enum llama_model_kv_override_type { + LLAMA_KV_OVERRIDE_TYPE_INT, + LLAMA_KV_OVERRIDE_TYPE_FLOAT, + LLAMA_KV_OVERRIDE_TYPE_BOOL, + LLAMA_KV_OVERRIDE_TYPE_STR, + }; + + enum llama_model_meta_key { + LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE, + LLAMA_MODEL_META_KEY_SAMPLING_TOP_K, + LLAMA_MODEL_META_KEY_SAMPLING_TOP_P, + LLAMA_MODEL_META_KEY_SAMPLING_MIN_P, + LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY, + LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD, + LLAMA_MODEL_META_KEY_SAMPLING_TEMP, + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N, + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA, + }; + + struct llama_model_kv_override { + enum llama_model_kv_override_type tag; + + char key[128]; + + union { + int64_t val_i64; + double val_f64; + bool val_bool; + char val_str[128]; + }; + }; + + struct llama_model_tensor_buft_override { + const char * pattern; + ggml_backend_buffer_type_t buft; + }; + + struct llama_model_params { + // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) + ggml_backend_dev_t * devices; + + // NULL-terminated list of buffer types to use for tensors that match a pattern + const struct llama_model_tensor_buft_override * tensor_buft_overrides; + + int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers + enum llama_split_mode split_mode; // how to split the model across multiple GPUs + + // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE + int32_t main_gpu; + + // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() + const float * tensor_split; + + // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. + // If the provided progress_callback returns true, model loading continues. + // If it returns false, model loading is immediately aborted. + llama_progress_callback progress_callback; + + // context pointer passed to the progress callback + void * progress_callback_user_data; + + // override key-value pairs of the model meta data + const struct llama_model_kv_override * kv_overrides; + + // Keep the booleans together to avoid misalignment during copy-by-value. + bool vocab_only; // only load the vocabulary, no weights + bool use_mmap; // use mmap if possible + bool use_direct_io; // use direct io, takes precedence over use_mmap when supported + bool use_mlock; // force system to keep model in RAM + bool check_tensors; // validate model tensor data + bool use_extra_bufts; // use extra buffer types (used for weight repacking) + bool no_host; // bypass host buffer allowing extra buffers to be used + bool no_alloc; // only load metadata and simulate memory allocations + }; + + struct llama_sampler_seq_config { + llama_seq_id seq_id; + struct llama_sampler * sampler; + }; + + // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations + // https://github.com/ggml-org/llama.cpp/pull/7544 + struct llama_context_params { + uint32_t n_ctx; // text context, 0 = from model + uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode + uint32_t n_ubatch; // physical maximum batch size + uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing + + enum llama_context_type ctx_type; // set the context type (e.g. MTP) + enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` + enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id + enum llama_attention_type attention_type; // attention type to use for embeddings + enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention + + // ref: https://github.com/ggml-org/llama.cpp/pull/2054 + float rope_freq_base; // RoPE base frequency, 0 = from model + float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model + float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model + float yarn_attn_factor; // YaRN magnitude scaling factor + float yarn_beta_fast; // YaRN low correction dim + float yarn_beta_slow; // YaRN high correction dim + uint32_t yarn_orig_ctx; // YaRN original context size + float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default) + + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; + + enum ggml_type type_k; // data type for K cache [EXPERIMENTAL] + enum ggml_type type_v; // data type for V cache [EXPERIMENTAL] + + // Abort callback + // if it returns true, execution of llama_decode() will be aborted + // currently works only with CPU execution + ggml_abort_callback abort_callback; + void * abort_callback_data; + + // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. + bool embeddings; // if true, extract embeddings (together with logits) + bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU + bool no_perf; // measure performance timings + bool op_offload; // offload host tensor operations to device + bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) + // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases + // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 + bool kv_unified; // use a unified buffer across the input sequences when computing the attention + // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix + // ref: https://github.com/ggml-org/llama.cpp/pull/14363 + + // [EXPERIMENTAL] + // backend sampler chain configuration (make sure the caller keeps the sampler chains alive) + // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init) + struct llama_sampler_seq_config * samplers; + size_t n_samplers; + + // a source/target/parent context + // can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts + struct llama_context * ctx_other; + }; + + struct llama_model_tensor_override { + const char * pattern; + enum ggml_type type; + }; + + struct llama_model_imatrix_data { + const char * name; + const float * data; + size_t size; + }; + + // model quantization parameters + typedef struct llama_model_quantize_params { + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // token embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + bool keep_split; // quantize to the same number of shards + bool dry_run; // calculate and show the final quantization size without performing quantization + const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data + const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides + const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides + const int32_t * prune_layers; // pointer to layer indices to prune + } llama_model_quantize_params; + + typedef struct llama_logit_bias { + llama_token token; + float bias; + } llama_logit_bias; + + typedef struct llama_sampler_chain_params { + bool no_perf; // whether to measure performance timings + } llama_sampler_chain_params; + + // used in chat template + typedef struct llama_chat_message { + const char * role; + const char * content; + } llama_chat_message; + + // lora adapter + struct llama_adapter_lora; + + // Helpers for getting default parameters + // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172) + LLAMA_API struct llama_model_params llama_model_default_params(void); + LLAMA_API struct llama_context_params llama_context_default_params(void); + LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void); + LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void); + + // Initialize the llama + ggml backend + // If numa is true, use NUMA optimizations + // Call once at the start of the program + LLAMA_API void llama_backend_init(void); + + // Call once at the end of the program - currently only used for MPI + LLAMA_API void llama_backend_free(void); + + //optional: + LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa); + + // Optional: an auto threadpool gets created in ggml if not passed explicitly + LLAMA_API void llama_attach_threadpool( + struct llama_context * ctx, + ggml_threadpool_t threadpool, + ggml_threadpool_t threadpool_batch); + + LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); + + typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata); + + // Create a new model from GGUF metadata as well as a function to set the tensor data + // - tensors are created as GGML_TYPE_F32 by default, + // override by adding a tensor with the same name but a different name to the context + LLAMA_API struct llama_model * llama_model_init_from_user( + struct gguf_context * metadata, + llama_model_set_tensor_data_t set_tensor_data, // function to initialize tensor data with + void * set_tensor_data_ud, // userdata for function + struct llama_model_params params); + + DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( + const char * path_model, + struct llama_model_params params), + "use llama_model_load_from_file instead"); + + // Load a model from a file + // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf + // If the split file name does not follow this pattern, use llama_model_load_from_splits + LLAMA_API struct llama_model * llama_model_load_from_file( + const char * path_model, + struct llama_model_params params); + + // Load a model from an open FILE pointer + LLAMA_API struct llama_model * llama_model_load_from_file_ptr( + FILE * file, + struct llama_model_params params); + + // Load a model from multiple splits (support custom naming scheme) + // The paths must be in the correct order + LLAMA_API struct llama_model * llama_model_load_from_splits( + const char ** paths, + size_t n_paths, + struct llama_model_params params); + + LLAMA_API void llama_model_save_to_file( + const struct llama_model * model, + const char * path_model); + + DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model), + "use llama_model_free instead"); + + LLAMA_API void llama_model_free(struct llama_model * model); + + LLAMA_API struct llama_context * llama_init_from_model( + struct llama_model * model, + struct llama_context_params params); + + DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model( + struct llama_model * model, + struct llama_context_params params), + "use llama_init_from_model instead"); + + // Frees all allocated memory + LLAMA_API void llama_free(struct llama_context * ctx); + + LLAMA_API int64_t llama_time_us(void); + + LLAMA_API size_t llama_max_devices(void); + LLAMA_API size_t llama_max_parallel_sequences(void); + LLAMA_API size_t llama_max_tensor_buft_overrides(void); + + LLAMA_API bool llama_supports_mmap (void); + LLAMA_API bool llama_supports_mlock (void); + LLAMA_API bool llama_supports_gpu_offload(void); + LLAMA_API bool llama_supports_rpc (void); + + // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions + // In some cases the requested values via llama_context_params may differ from the actual values used by the context + // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732 + LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx); + + DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); + DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); + DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); + DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); + + DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); + + LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); + LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx); + LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type + + LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); + LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model); + + LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model); + + // Get the model's RoPE frequency scaling factor + LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); + + // Returns the number of classifier outputs (only valid for classifier models) + // Undefined behavior for non-classifier models + LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); + + // Returns label of classifier output by index ( 1` + // p0 < 0 : [0, p1] + // p1 < 0 : [p0, inf) + LLAMA_API void llama_memory_seq_div( + llama_memory_t mem, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d); + + // Returns the smallest position present in the memory for the specified sequence + // This is typically non-zero only for SWA caches + // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory + // Return -1 if the sequence is empty + LLAMA_API llama_pos llama_memory_seq_pos_min( + llama_memory_t mem, + llama_seq_id seq_id); + + // Returns the largest position present in the memory for the specified sequence + // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory + // Return -1 if the sequence is empty + LLAMA_API llama_pos llama_memory_seq_pos_max( + llama_memory_t mem, + llama_seq_id seq_id); + + // Check if the memory supports shifting + LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); + + // + // State / sessions + // + + // Returns the *actual* size in bytes of the state + // (logits, embedding and memory) + // Only use when saving the state, not when restoring it, otherwise the size may be too small. + LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); + LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx), + "use llama_state_get_size instead"); + + // Copies the state to the specified destination address. + // Destination needs to have allocated enough memory. + // Returns the number of bytes copied + LLAMA_API size_t llama_state_get_data( + struct llama_context * ctx, + uint8_t * dst, + size_t size); + LLAMA_API DEPRECATED(size_t llama_copy_state_data( + struct llama_context * ctx, + uint8_t * dst), + "use llama_state_get_data instead"); + + // Set the state reading from the specified address + // Returns the number of bytes read + LLAMA_API size_t llama_state_set_data( + struct llama_context * ctx, + const uint8_t * src, + size_t size); + LLAMA_API DEPRECATED(size_t llama_set_state_data( + struct llama_context * ctx, + const uint8_t * src), + "use llama_state_set_data instead"); + + // Save/load session file + LLAMA_API bool llama_state_load_file( + struct llama_context * ctx, + const char * path_session, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + LLAMA_API DEPRECATED(bool llama_load_session_file( + struct llama_context * ctx, + const char * path_session, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out), + "use llama_state_load_file instead"); + + LLAMA_API bool llama_state_save_file( + struct llama_context * ctx, + const char * path_session, + const llama_token * tokens, + size_t n_token_count); + LLAMA_API DEPRECATED(bool llama_save_session_file( + struct llama_context * ctx, + const char * path_session, + const llama_token * tokens, + size_t n_token_count), + "use llama_state_save_file instead"); + + // Get the exact size needed to copy the state of a single sequence + LLAMA_API size_t llama_state_seq_get_size( + struct llama_context * ctx, + llama_seq_id seq_id); + + // Copy the state of a single sequence into the specified buffer + LLAMA_API size_t llama_state_seq_get_data( + struct llama_context * ctx, + uint8_t * dst, + size_t size, + llama_seq_id seq_id); + + // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence + // Returns: + // - Positive: Ok + // - Zero: Failed to load + LLAMA_API size_t llama_state_seq_set_data( + struct llama_context * ctx, + const uint8_t * src, + size_t size, + llama_seq_id dest_seq_id); + + LLAMA_API size_t llama_state_seq_save_file( + struct llama_context * ctx, + const char * filepath, + llama_seq_id seq_id, + const llama_token * tokens, + size_t n_token_count); + + LLAMA_API size_t llama_state_seq_load_file( + struct llama_context * ctx, + const char * filepath, + llama_seq_id dest_seq_id, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + +#define LLAMA_STATE_SEQ_FLAGS_NONE 0 + +// for backwards-compat +#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 + +// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) +#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 + +// Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load). +// Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag. +#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 + + typedef uint32_t llama_state_seq_flags; + + LLAMA_API size_t llama_state_seq_get_size_ext( + struct llama_context * ctx, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_get_data_ext( + struct llama_context * ctx, + uint8_t * dst, + size_t size, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_set_data_ext( + struct llama_context * ctx, + const uint8_t * src, + size_t size, + llama_seq_id dest_seq_id, + llama_state_seq_flags flags); + + // + // Decoding + // + + // Return batch for single sequence of tokens + // The sequence ID will be fixed to 0 + // The position of the tokens will be tracked automatically by llama_decode + // + // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it + // + LLAMA_API struct llama_batch llama_batch_get_one( + llama_token * tokens, + int32_t n_tokens); + + // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens + // Each token can be assigned up to n_seq_max sequence ids + // The batch has to be freed with llama_batch_free() + // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) + // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token + // The rest of the llama_batch members are allocated with size n_tokens + // All members are left uninitialized + LLAMA_API struct llama_batch llama_batch_init( + int32_t n_tokens, + int32_t embd, + int32_t n_seq_max); + + // Frees a batch of tokens allocated with llama_batch_init() + LLAMA_API void llama_batch_free(struct llama_batch batch); + + // Process a batch of tokens. + // In contrast to llama_decode() - this call does not use KV cache. + // For encode-decoder contexts, processes the batch using the encoder. + // Can store the encoder output internally for later use by the decoder's cross-attention layers. + // 0 - success + // < 0 - error. the memory state is restored to the state before this call + LLAMA_API int32_t llama_encode( + struct llama_context * ctx, + struct llama_batch batch); + + // Process a batch of tokens. + // Requires the context to have a memory. + // For encode-decoder contexts, processes the batch using the decoder. + // Positive return values does not mean a fatal error, but rather a warning. + // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context + // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + // Upon other return values, the memory state is restored to the state before this call + // 0 - success + // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) + // 2 - aborted (processed ubatches will remain in the context's memory) + // -1 - invalid input batch + // < -1 - fatal error (processed ubatches will remain in the context's memory) + LLAMA_API int32_t llama_decode( + struct llama_context * ctx, + struct llama_batch batch); + + // Set the number of threads used for decoding + // n_threads is the number of threads used for generation (single token) + // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) + LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch); + + // Get the number of threads used for generation of a single token. + LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); + + // Get the number of threads used for prompt and batch processing (multiple token). + LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); + + // Set whether the context outputs embeddings or not + // TODO: rename to avoid confusion with llama_get_embeddings() + LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); + + // Set whether to use causal attention or not + // If set to true, the model will only attend to the past tokens + LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); + + // Set whether the model is in warmup mode or not + // If true, all model tensors are activated during llama_decode() to load and cache their weights. + // + // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models, + // so it is generally not recommended to use in practice. will be removed in the future + DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup), + "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]"); + + // Set abort callback + LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); + + // Wait until all computations are finished + // This is automatically done when using one of the functions below to obtain the computation results + // and is not necessary to call it explicitly in most cases + LLAMA_API void llama_synchronize(struct llama_context * ctx); + + // Token logits obtained from the last call to llama_decode() + // The logits for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have appeared in the batch. + // Rows: number of tokens for which llama_batch.logits[i] != 0 + // Cols: n_vocab + // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) + LLAMA_API float * llama_get_logits(struct llama_context * ctx); + + // Logits for the ith token. For positive indices, Equivalent to: + // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab + // Negative indices can be used to access logits in reverse order, -1 is the last logit. + // returns NULL for invalid ids. + LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); + + // Get all output token embeddings. + // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, + // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have appeared in the batch. + // shape: [n_outputs*n_embd] + // Otherwise, returns NULL. + // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) + LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); + + // Get the embeddings for the ith token. For positive indices, Equivalent to: + // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd + // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding. + // shape: [n_embd] (1-dimensional) + // returns NULL for invalid ids. + LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); + + // Get the embeddings for a sequence id + // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE + // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence + // otherwise: float[n_embd] (1-dimensional) + LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); + + // + // backend sampling API [EXPERIMENTAL] + // note: use only if the llama_context was created with at least one llama_sampler_seq_config + // + + // Get the backend sampled token for the ith token. + // Returns LLAMA_TOKEN_NULL if no token was sampled. + LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled probabilities for the ith token + // The index matches llama_get_sampled_token_ith(). + // Returns NULL if no probabilities were generated. + LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled logits for the ith token + // Returns NULL if no logits were sampled. + LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled candidates (token ids) for the ith token + // These are needed to map probability/logit indices to vocab token ids. + // Returns NULL if no candidates were sampled. + LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i); + + // + // Vocab + // + + LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token); + + LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token); + + LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token); + + // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) + LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token); + + // Identify if Token Id is a control token or a render-able token + LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token); + + // Special tokens + LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence + LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence + LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn + LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator + LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line + LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding + LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask + + LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab); + LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab); + LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab); + + LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab); + + DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); + DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); + DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); + DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); + DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); + DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); + DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); + DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); + DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); + DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); + DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); + DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); + + // CLS is equivalent to BOS + DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification + "use llama_vocab_bos instead"); + + // + // Tokenization + // + // The API is thread-safe. + // + + /// @details Convert the provided text into tokens. + /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. + /// @return Returns the number of tokens on success, no more than n_tokens_max + /// @return Returns a negative number on failure - the number of tokens that would have been returned + /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit) + /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so. + /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated + /// as plaintext. Does not insert a leading space. + LLAMA_API int32_t llama_tokenize( + const struct llama_vocab * vocab, + const char * text, + int32_t text_len, + llama_token * tokens, + int32_t n_tokens_max, + bool add_special, + bool parse_special); + + // Token Id -> Piece. + // Uses the vocabulary in the provided context. + // Does not write null terminator to the buffer. + // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') + // @param special If true, special tokens are rendered in the output. + LLAMA_API int32_t llama_token_to_piece( + const struct llama_vocab * vocab, + llama_token token, + char * buf, + int32_t length, + int32_t lstrip, + bool special); + + /// @details Convert the provided tokens into text (inverse of llama_tokenize()). + /// @param text The char pointer must be large enough to hold the resulting text. + /// @return Returns the number of chars/bytes on success, no more than text_len_max. + /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned. + /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. + /// @param unparse_special If true, special tokens are rendered in the output. + LLAMA_API int32_t llama_detokenize( + const struct llama_vocab * vocab, + const llama_token * tokens, + int32_t n_tokens, + char * text, + int32_t text_len_max, + bool remove_special, + bool unparse_special); + + // + // Chat templates + // + + /// Apply chat template. Inspired by hf apply_chat_template() on python. + /// + /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template + /// @param tmpl A Jinja template to use for this chat. + /// @param chat Pointer to a list of multiple llama_chat_message + /// @param n_msg Number of llama_chat_message in this chat + /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. + /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages) + /// @param length The size of the allocated buffer + /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. + LLAMA_API int32_t llama_chat_apply_template( + const char * tmpl, + const struct llama_chat_message * chat, + size_t n_msg, + bool add_ass, + char * buf, + int32_t length); + + // Get list of built-in chat templates + LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len); + + // + // Sampling API + // + // Sample usage: + // + // // prepare the sampling chain at the start + // auto sparams = llama_sampler_chain_default_params(); + // + // llama_sampler * smpl = llama_sampler_chain_init(sparams); + // + // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50)); + // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); + // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8)); + // + // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat" + // // this sampler will be responsible to select the actual token + // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); + // + // ... + // + // // decoding loop: + // while (...) { + // ... + // + // llama_decode(ctx, batch); + // + // // sample from the logits of the last token in the batch + // const llama_token id = llama_sampler_sample(smpl, ctx, -1); + // + // ... + // } + // + // llama_sampler_free(smpl); + // + + typedef void * llama_sampler_context_t; + + struct llama_sampler_data { + struct ggml_tensor * logits; + struct ggml_tensor * probs; + struct ggml_tensor * sampled; + struct ggml_tensor * candidates; + }; + + // user code can implement the interface below in order to create custom llama_sampler + struct llama_sampler_i { + const char * (*name) (const struct llama_sampler * smpl); // can be NULL + void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL + void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required + void (*reset) ( struct llama_sampler * smpl); // can be NULL + struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL + void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL + + // [EXPERIMENTAL] + // backend sampling interface: + + // return true if the backend supports all ops needed by the sampler + // note: call once per sampler + bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft); + + // call after .backend_apply() + void (*backend_accept)( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct ggml_tensor * selected_token); + + // call after .backend_init() + void (*backend_apply)( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct llama_sampler_data * data); + + // called before graph execution to set inputs for the current ubatch + void (*backend_set_input)(struct llama_sampler * smpl); + }; + + struct llama_sampler { + struct llama_sampler_i * iface; + + llama_sampler_context_t ctx; + }; + + // [EXPERIMENTAL] + // attach a sampler to the context + // note: prefer initializing the context with llama_context_params.samplers when possible + LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl); + + // mirror of llama_sampler_i: + LLAMA_API struct llama_sampler * llama_sampler_init ( struct llama_sampler_i * iface, llama_sampler_context_t ctx); + LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); + LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); + LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); + LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); + LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); + // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) + LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); + + // llama_sampler_chain + // a type of llama_sampler that can chain multiple samplers one after another + + LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params); + + // important: takes ownership of the sampler object and will free it when llama_sampler_free is called + LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl); + + // return NULL if: + // - the sampler is NULL + // - the sampler is not a llama_sampler_chain + // - the index is out of bounds, unless i == -1 + // - if i == -1, returns the chain itself (can be used to check if the sampler is a chain) + LLAMA_API struct llama_sampler * llama_sampler_chain_get( struct llama_sampler * chain, int32_t i); + + // the total number of samplers in the chain + LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); + + // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed + LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i); + + // available samplers: + + LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void); + + /// seed == LLAMA_DEFAULT_SEED to use a random seed. + LLAMA_API struct llama_sampler * llama_sampler_init_dist(uint32_t seed); + + /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// Setting k <= 0 makes this a noop + LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); + + /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep); + + /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 + LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep); + + /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep); + + /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf + LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); + + /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. + LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent); + + /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 + LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed); + + /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 + LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n); + + /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API struct llama_sampler * llama_sampler_init_mirostat( + int32_t n_vocab, + uint32_t seed, + float tau, + float eta, + int32_t m); + + /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2( + uint32_t seed, + float tau, + float eta); + + /// @details Initializes a GBNF grammar, see grammars/README.md for details. + /// @param vocab The vocabulary that this grammar will be used with. + /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. + /// @param grammar_root The name of the start symbol for the grammar. + LLAMA_API struct llama_sampler * llama_sampler_init_grammar( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root); + + DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root, + const char ** trigger_words, + size_t num_trigger_words, + const llama_token * trigger_tokens, + size_t num_trigger_tokens), + "use llama_sampler_init_grammar_lazy_patterns instead"); + + + /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 + /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. + /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. + LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root, + const char ** trigger_patterns, + size_t num_trigger_patterns, + const llama_token * trigger_tokens, + size_t num_trigger_tokens); + + + /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. + LLAMA_API struct llama_sampler * llama_sampler_init_penalties( + int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) + float penalty_repeat, // 1.0 = disabled + float penalty_freq, // 0.0 = disabled + float penalty_present); // 0.0 = disabled + + /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 + LLAMA_API struct llama_sampler * llama_sampler_init_dry( + const struct llama_vocab * vocab, + int32_t n_ctx_train, + float dry_multiplier, + float dry_base, + int32_t dry_allowed_length, + int32_t dry_penalty_last_n, + const char ** seq_breakers, + size_t num_breakers); + + /// adaptive-p: select tokens near a configurable target probability over time. + /// + /// the adaptive-p sampler transforms the token probability distribution to favor tokens + /// that fall near a user-configurable probability target. + /// + /// internally, the sampler maintains an exponential moving average of the *ORIGINAL* + /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an + /// adapted target probability at each sampling step, thus maintaining the desired target + /// probability over time. + /// + /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last + /// in the sampler chain (like mirostat, dist, greedy). + /// + /// only mild truncation before this sampler is recommended. we suggest applying min-p + /// before adaptive-p as the only other active sampler in the chain. + /// + /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) + /// @param decay EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99) + /// @param seed RNG seed + /// + /// ref: https://github.com/ggml-org/llama.cpp/pull/17927 + /// + LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p( + float target, + float decay, + uint32_t seed); + + LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( + int32_t n_vocab, + int32_t n_logit_bias, + const llama_logit_bias * logit_bias); + + // this sampler is meant to be used for fill-in-the-middle infilling + // it's supposed to be used after top_k + top_p sampling + // + // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG + // 2. combine probs of tokens that have the same prefix + // + // example: + // + // - before: + // "hel": 0.5 + // "hell": 0.2 + // "hello": 0.1 + // "dummy": 0.1 + // + // - after: + // "hel": 0.8 + // "dummy": 0.1 + // + // 3. discard non-EOG tokens with low prob + // 4. if no tokens are left -> pick EOT + // + LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab); + + // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise + LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl); + + /// @details Sample and accept a token from the idx-th output of the last evaluation + // + // Shorthand for: + // const auto * logits = llama_get_logits_ith(ctx, idx); + // llama_token_data_array cur_p = { ... init from logits ... }; + // llama_sampler_apply(smpl, &cur_p); + // auto token = cur_p.data[cur_p.selected].id; + // llama_sampler_accept(smpl, token); + // return token; + // Returns the sampled token + LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); + + // TODO: extend in the future + //LLAMA_API void llama_decode_with_sampler(struct llama_context * ctx, struct llama_sampler * smpl, struct llama_batch batch, ...); + + // + // Model split + // + + /// @details Build a split GGUF final path for this chunk. + /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" + // Returns the split_path length. + LLAMA_API int32_t llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int32_t split_no, int32_t split_count); + + /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. + /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" + // Returns the split_prefix length. + LLAMA_API int32_t llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int32_t split_no, int32_t split_count); + + // Print system information + LLAMA_API const char * llama_print_system_info(void); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + // The logger state is global so these functions are NOT thread safe. + LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data); + LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); + + // + // Performance utils + // + // NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements. + // + + struct llama_perf_context_data { + // ms == milliseconds + double t_start_ms; // absolute start time + double t_load_ms; // time needed for loading the model + double t_p_eval_ms; // time needed for processing the prompt + double t_eval_ms; // time needed for generating tokens + + int32_t n_p_eval; // number of prompt tokens + int32_t n_eval; // number of generated tokens + int32_t n_reused; // number of times a ggml compute graph had been reused + }; + + struct llama_perf_sampler_data { + double t_sample_ms; // time needed for sampling in ms + + int32_t n_sample; // number of sampled tokens + }; + + LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx); + LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); + LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); + + // NOTE: the following work only with samplers constructed via llama_sampler_chain_init + LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain); + LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); + LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); + + // + // training + // + + // function that returns whether or not a given tensor contains trainable parameters + typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata); + + // always returns true + LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata); + + struct llama_opt_params { + uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0 + + llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters + void * param_filter_ud; // userdata for determining which tensors contain trainable parameters + + ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters + void * get_opt_pars_ud; // userdata for calculating optimizer parameters + + enum ggml_opt_optimizer_type optimizer_type; + }; + + LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params); + + LLAMA_API void llama_opt_epoch( + struct llama_context * lctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, + ggml_opt_result_t result_eval, + int64_t idata_split, + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval); + +#ifdef __cplusplus +} +#endif + +#endif // LLAMA_H diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/mtmd-helper.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/mtmd-helper.h new file mode 100644 index 00000000..164b7c66 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/mtmd-helper.h @@ -0,0 +1,176 @@ +#ifndef MTMD_HELPER_H +#define MTMD_HELPER_H + +#include "ggml.h" +#include "llama.h" +#include "mtmd.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// +// libmtmd helper functions +// +// Please note that these helpers are not guaranteed to be stable. +// BREAKING CHANGES are expected. +// + +struct mtmd_helper_video; +typedef struct mtmd_helper_video mtmd_helper_video; + +// Set callback for all future logging events. +// If this is not called, or NULL is supplied, everything is output on stderr. +// Note: this also call mtmd_log_set() internally +MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data); + +// Returns true if this build includes video support (MTMD_VIDEO was ON at compile time). +MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx); + +struct mtmd_helper_bitmap_wrapper { + mtmd_bitmap * bitmap; + mtmd_helper_video * video_ctx; +}; + +// helper function to construct a mtmd_bitmap from a file +// it calls mtmd_helper_bitmap_init_from_buf() internally +// returns nullptr on failure +// this function is thread-safe +MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); + +// helper function to construct a mtmd_bitmap from a buffer containing a file +// supported formats: +// image: formats supported by stb_image: jpg, png, bmp, gif, etc. +// audio: formats supported by miniaudio: wav, mp3, flac +// note: +// - for now, video input is only supported via C++ helper functions +// - audio files will be auto-detected based on magic bytes +// - output bitmap will have FNV hash as the ID +// returns nullptr on failure +// this function is thread-safe +MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); + +// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache +MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); + +// helper to count the total position of tokens from a list of chunks, useful to keep track of n_past +// normally, n_pos is equal to n_tokens, but for M-RoPE it is different +MTMD_API llama_pos mtmd_helper_get_n_pos(const mtmd_input_chunks * chunks); + +// helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE +// out_pos must have length == mtmd_helper_get_n_tokens(image) +MTMD_API void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * image, llama_pos pos_0, struct mtmd_decoder_pos * out_pos); + +// helper function that automatically: +// 1. run llama_decode() on text chunks +// 2. run mtmd_encode() on image chunks, then mtmd_get_output_embd() and then llama_decode() +// if any of the mtmd_encode() or llama_decode() calls return non-zero, stop and forward the error +// otherwise, returns 0 on success +// this function is NOT thread-safe +MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunks * chunks, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + bool logits_last, + llama_pos * new_n_past); + +// works like mtmd_helper_eval_chunks(), but only for a single chunk +// this function is NOT thread-safe +MTMD_API int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunk * chunk, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + bool logits_last, + llama_pos * new_n_past); + +// helper function to decode an image whose embeddings have already been calculated +// this helper will handle batching and pre/post decoding setup (for ex. gemma 3 requires non-causal attention) +// ret 0 on success, -1 on chunk not being a valid image chunk, 1 on decode failure +MTMD_API int32_t mtmd_helper_decode_image_chunk(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunk * chunk, + float * encoded_embd, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + llama_pos * new_n_past); + +// +// video input helpers (requires ffmpeg/ffprobe installed on the system) +// the notion of video only exists at the helper level, it is not visible to the core mtmd library +// +// NOTE: this implementation is model-agnostic, it can be used with any vision-capable model +// however, it may not be accurate for some specific models +// (this is expected for now, to keep the implementation simple) +// + +struct mtmd_helper_video_info { + uint32_t width; + uint32_t height; + float fps; // effective fps (fps_target if set, else original video fps) + int32_t n_frames; // estimated total frames at effective fps (-1 if unknown) +}; + +struct mtmd_helper_video_init_params { + float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f + const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH + int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms + // TODO @ngxson : allow "placeholder" bitmap output for counting tokens +}; + +MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void); + +// returns NULL on failure (ffprobe not found, file unreadable, etc.) +MTMD_API mtmd_helper_video * mtmd_helper_video_init( + struct mtmd_context * mctx, + const char * path, + struct mtmd_helper_video_init_params params); + +// Same as mtmd_helper_video_init(), but reads from an in-memory buffer. +// The buffer is copied internally; the caller does not need to keep it alive. +// Note: pipe input is not seekable, so seeking will use output-side seeking +// (ffmpeg decodes and discards frames up to the target position). +MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( + struct mtmd_context * mctx, + const unsigned char * buf, size_t len, + struct mtmd_helper_video_init_params params); +MTMD_API void mtmd_helper_video_free(mtmd_helper_video * ctx); +MTMD_API struct mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx); + +// Read the next item from the video stream; exactly one of out_bitmap or out_text is set per call. +// *out_bitmap - heap-allocated; caller must free with mtmd_bitmap_free() +// *out_text - heap-allocated (always via strdup/malloc); caller must free with free() +// returns 0 on success, -1 on EOF, -2 on error +MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, + mtmd_bitmap ** out_bitmap, + char ** out_text); + +#ifdef __cplusplus +} // extern "C" +#endif + +// +// C++ wrappers +// + +#ifdef __cplusplus +namespace mtmd_helper { + +// video-related C++ wrappers +struct mtmd_helper_video_deleter { + void operator()(mtmd_helper_video * val) { mtmd_helper_video_free(val); } +}; +using video_ptr = std::unique_ptr; + +} // namespace mtmd_helper +#endif + +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/mtmd.h b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/mtmd.h new file mode 100644 index 00000000..a76a6ec2 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Headers/mtmd.h @@ -0,0 +1,387 @@ +#ifndef MTMD_H +#define MTMD_H + +#include "ggml.h" +#include "llama.h" + +#include +#include +#include + +#ifdef __cplusplus +#include +#include +#include +#include +#include +#endif + +/** + * libmtmd: A library for multimodal support in llama.cpp. + * + * WARNING: This API is experimental and subject to many BREAKING CHANGES. + * Issues related to API usage may receive lower priority support. + * + * For the usage, see an example in mtmd-cli.cpp + * + * For contributors: + * - Make sure the C API is aligned with the libllama C API (as in llama.h) + * - Do not include model name (e.g., qwen, gemma) in the API, use generic terms instead + * - Keep the API minimal, do not expose internal details unless necessary + * + * IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated. + * We encourage human contributors to ensure the quality and reliability of the codebase. + */ + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define MTMD_API __declspec(dllexport) +# else +# define MTMD_API __declspec(dllimport) +# endif +# else +# define MTMD_API __attribute__ ((visibility ("default"))) +# endif +#else +# define MTMD_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +enum mtmd_input_chunk_type { + MTMD_INPUT_CHUNK_TYPE_TEXT, + MTMD_INPUT_CHUNK_TYPE_IMAGE, + MTMD_INPUT_CHUNK_TYPE_AUDIO, +}; + +// opaque types +struct mtmd_context; +struct mtmd_bitmap; +struct mtmd_image_tokens; +struct mtmd_input_chunk; +struct mtmd_input_chunks; + +struct mtmd_input_text { + const char * text; + bool add_special; + bool parse_special; +}; + +// +// C API +// + +typedef struct mtmd_context mtmd_context; +typedef struct mtmd_bitmap mtmd_bitmap; +typedef struct mtmd_image_tokens mtmd_image_tokens; +typedef struct mtmd_input_chunk mtmd_input_chunk; +typedef struct mtmd_input_chunks mtmd_input_chunks; +typedef struct mtmd_input_text mtmd_input_text; + +struct mtmd_context_params { + bool use_gpu; + bool print_timings; + int n_threads; + const char * image_marker; // deprecated, use media_marker instead + const char * media_marker; + enum llama_flash_attn_type flash_attn_type; + bool warmup; // whether to run a warmup encode pass after initialization + + // limit number of image tokens, only for vision models with dynamic resolution + int image_min_tokens; // minimum number of tokens for image input (default: read from metadata) + int image_max_tokens; // maximum number of tokens for image input (default: read from metadata) + + // callback function passed over to mtmd proper + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; +}; + +MTMD_API const char * mtmd_default_marker(void); + +MTMD_API struct mtmd_context_params mtmd_context_params_default(void); + +// initialize the mtmd context +// return nullptr on failure +MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname, + const struct llama_model * text_model, + const struct mtmd_context_params ctx_params); + +MTMD_API void mtmd_free(mtmd_context * ctx); + +// whether we need to set non-causal mask before llama_decode +// if chunk is nullptr, we assume the default case where chunk is an image chunk +MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk); + +// whether the current model use M-RoPE for llama_decode +MTMD_API bool mtmd_decode_use_mrope(const mtmd_context * ctx); + +// whether the current model supports vision input +MTMD_API bool mtmd_support_vision(const mtmd_context * ctx); + +// whether the current model supports audio input +MTMD_API bool mtmd_support_audio(const mtmd_context * ctx); + +// get audio sample rate in Hz, for example 16000 for Whisper +// return -1 if audio is not supported +MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); + +// get the current marker string +MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); + +// mtmd_bitmap +// +// if bitmap is image: +// length of data must be nx * ny * 3 +// the data is in RGBRGBRGB... format +// note: some video-capable models (i.e. qwen-vl) can merge consecutive bitmaps +// into one chunk, mtmd_tokenize() will automatically handle this +// if bitmap is audio: +// length of data must be n_samples * sizeof(float) +// the data is in float format (PCM F32) +// +// if data == nullptr: +// the bitmap is considered "empty", and will be treated as a placeholder for counting tokens +// you can pass the bitmap via mtmd_tokenize(), then call mtmd_*_get_n_tokens() to count the tokens +// note: passing a placeholder bitmap to mtmd_encode() will return an error +MTMD_API mtmd_bitmap * mtmd_bitmap_init (uint32_t nx, uint32_t ny, const unsigned char * data); +MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); +MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap); +MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); +MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap); +MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); +MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); +// bitmap ID is optional, but useful for KV cache tracking +// these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data() +MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap); +MTMD_API void mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id); + +// mtmd_bitmap lazy +// +// this is a special bitmap that: +// - does not hold the actual data +// - can be expanded into one or more chunks (either media to text chunks) +// user must provide a callback to fill in the data when mtmd_tokenize() is called +// this is useful for large video inputs: +// - allow reading video frame by frame, without loading the entire video into memory +// - allow tracking the whole video with a single ID (for example, the file hash) + +// set (*out_bitmap) to non-nullptr to emit a bitmap chunk; it will be freed automatically +// set (*out_text) to non-nullptr to emit a text chunk; it must be heap-allocated, null-terminated and will be freed automatically +// either out_bitmap or out_text can be set, but not both +// out_bitmap cannot be another lazy bitmap (no nested lazy allowed) +// return value: +// 0 on success +// -1 on EOF (signal to mtmd_tokenize to move on) +// -2 on error (signal to mtmd_tokenize to abort) +typedef int(* mtmd_bitmap_lazy_callback)( + size_t chunk_idx, + void * user_data, + mtmd_bitmap ** out_bitmap, + char ** out_text); + +MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx, + const char * id, // usually set to file hash + void * user_data, + mtmd_bitmap_lazy_callback callback); + +// mtmd_input_chunks +// +// this is simply a list of mtmd_input_chunk +// the elements can only be populated via mtmd_tokenize() +MTMD_API mtmd_input_chunks * mtmd_input_chunks_init(void); +MTMD_API size_t mtmd_input_chunks_size(const mtmd_input_chunks * chunks); +MTMD_API const mtmd_input_chunk * mtmd_input_chunks_get (const mtmd_input_chunks * chunks, size_t idx); +MTMD_API void mtmd_input_chunks_free(mtmd_input_chunks * chunks); + +// mtmd_input_chunk +// +// the instance will be constructed via mtmd_tokenize() +// it will be freed along with mtmd_input_chunks +MTMD_API enum mtmd_input_chunk_type mtmd_input_chunk_get_type (const mtmd_input_chunk * chunk); +MTMD_API const llama_token * mtmd_input_chunk_get_tokens_text (const mtmd_input_chunk * chunk, size_t * n_tokens_output); +MTMD_API const mtmd_image_tokens * mtmd_input_chunk_get_tokens_image(const mtmd_input_chunk * chunk); +MTMD_API size_t mtmd_input_chunk_get_n_tokens (const mtmd_input_chunk * chunk); +// returns nullptr for ID on text chunk +MTMD_API const char * mtmd_input_chunk_get_id (const mtmd_input_chunk * chunk); +// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise) +MTMD_API llama_pos mtmd_input_chunk_get_n_pos (const mtmd_input_chunk * chunk); + +// in case you want to use custom logic to handle the chunk (i.e. KV cache management) +// you can move the chunk ownership to your own code by copying it +// remember to free the chunk when you are done with it +MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk); +MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk); + + +// mtmd_image_tokens +// +// the instance will be constructed via mtmd_tokenize() +// it will be freed along with mtmd_input_chunk +MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); // TODO: deprecate +MTMD_API const char * mtmd_image_tokens_get_id (const mtmd_image_tokens * image_tokens); // TODO: deprecate +// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise) +MTMD_API llama_pos mtmd_image_tokens_get_n_pos (const mtmd_image_tokens * image_tokens); // TODO: deprecate + +DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens), + "use mtmd_image_tokens_get_decoder_pos() instead"); +DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens), + "use mtmd_image_tokens_get_decoder_pos() instead"); + +struct mtmd_decoder_pos { + uint32_t t; + uint32_t x; + uint32_t y; + uint32_t z; // unused for now, reserved for future use +}; +// get position for decoder attention, to be used by M-RoPE models +// i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 +// pos_0 is the absolute position of the first token +// return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position) +MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i); + +// tokenize an input text prompt and a list of bitmaps (images/audio) +// the prompt must have the input image marker (default: "<__media__>") in it +// the default marker is defined by mtmd_default_marker() +// the marker will be replaced with the image/audio chunk +// for example: +// "here is an image: <__media__>\ndescribe it in detail." +// this will gives 3 chunks: +// 1. "here is an image: " +// 2. (image/audio tokens) +// 3. "\ndescribe it in detail." +// number of bitmaps must be equal to the number of markers in the prompt +// this function is thread-safe (shared ctx) +// return values: +// 0 on success +// 1 on number of bitmaps not matching the number of markers +// 2 on image preprocessing error +MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, + mtmd_input_chunks * output, + const mtmd_input_text * text, + const mtmd_bitmap ** bitmaps, + size_t n_bitmaps); + +// returns 0 on success +// TODO: deprecate +MTMD_API int32_t mtmd_encode(mtmd_context * ctx, + const mtmd_image_tokens * image_tokens); + +// returns 0 on success +MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx, + const mtmd_input_chunk * chunk); + +// get output embeddings from the last encode pass +// the reading size (in bytes) is equal to: +// llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float) +MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx); + +// Set callback for all future logging events. +// If this is not called, or NULL is supplied, everything is output on stderr. +MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); + +// EXPERIMENTAL API to get mmproj's capabilities without initializing the full context +// This is only intended to be used by llama-server, breaking changes is expected +struct mtmd_caps { + bool inp_vision; + bool inp_audio; +}; +MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname); + +///////////////////////////////////////// + +// test function, to be used in test-mtmd-c-api.c +MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +// Get memory usage of the current model in bytes, per backend device +// Note: this is an unstable API, used internally by fit_params; it WILL be removed or changed without deprecation +#ifdef __cplusplus +MTMD_API std::map mtmd_get_memory_usage( + const char * mmproj_fname, + struct mtmd_context_params ctx_params); +#endif + +// +// C++ wrappers +// + +#ifdef __cplusplus + +namespace mtmd { + +struct mtmd_context_deleter { + void operator()(mtmd_context * val) { mtmd_free(val); } +}; +using context_ptr = std::unique_ptr; + +struct mtmd_bitmap_deleter { + void operator()(mtmd_bitmap * val) { mtmd_bitmap_free(val); } +}; +using bitmap_ptr = std::unique_ptr; + +struct mtmd_input_chunks_deleter { + void operator()(mtmd_input_chunks * val) { mtmd_input_chunks_free(val); } +}; +using input_chunks_ptr = std::unique_ptr; + +struct mtmd_input_chunk_deleter { + void operator()(mtmd_input_chunk * val) { mtmd_input_chunk_free(val); } +}; +using input_chunk_ptr = std::unique_ptr; + +struct bitmap { + bitmap_ptr ptr; + bitmap() : ptr(nullptr) {} + bitmap(mtmd_bitmap * bitmap) : ptr(bitmap) {} + bitmap(bitmap && other) noexcept : ptr(std::move(other.ptr)) {} + bitmap(uint32_t nx, uint32_t ny, const unsigned char * data) { + ptr.reset(mtmd_bitmap_init(nx, ny, data)); + } + ~bitmap() = default; + uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } + uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } + const unsigned char * data() const { return mtmd_bitmap_get_data(ptr.get()); } + size_t n_bytes() const { return mtmd_bitmap_get_n_bytes(ptr.get()); } + std::string id() const { return mtmd_bitmap_get_id(ptr.get()); } + void set_id(const char * id) const { mtmd_bitmap_set_id(ptr.get(), id); } +}; + +struct bitmaps { + std::vector entries; + ~bitmaps() = default; + // return list of pointers to mtmd_bitmap + // example: + // auto bitmaps_c_ptr = bitmaps.c_ptr(); + // int32_t res = mtmd_tokenize(... bitmaps_c_ptr.data(), bitmaps_c_ptr.size()); + std::vector c_ptr() { + std::vector res(entries.size()); + for (size_t i = 0; i < entries.size(); i++) { + res[i] = entries[i].ptr.get(); + } + return res; + } +}; + +struct input_chunks { + input_chunks_ptr ptr; + input_chunks() = default; + input_chunks(mtmd_input_chunks * chunks) : ptr(chunks) {} + ~input_chunks() = default; + size_t size() const { return mtmd_input_chunks_size(ptr.get()); } + const mtmd_input_chunk * operator[](size_t idx) const { + return mtmd_input_chunks_get(ptr.get(), idx); + } +}; + +} // namespace mtmd + +#endif + +#endif diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Info.plist b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Info.plist new file mode 100644 index 00000000..b5f25cf6 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Info.plist @@ -0,0 +1,16 @@ + + + + + CFBundleDevelopmentRegionen + CFBundleExecutableLlama + CFBundleIdentifierorg.ggml.llama + CFBundleInfoDictionaryVersion6.0 + CFBundleNameLlama + CFBundlePackageTypeFMWK + CFBundleShortVersionString0.9.0 + CFBundleVersion1 + MinimumOSVersion14.0 + + diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Llama b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Llama new file mode 100755 index 00000000..36e6ebb0 Binary files /dev/null and b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Llama differ diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Modules/module.modulemap b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Modules/module.modulemap new file mode 100644 index 00000000..4d610a57 --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/Modules/module.modulemap @@ -0,0 +1,5 @@ +framework module Llama { + umbrella header "llama.h" + export * + module * { export * } +} diff --git a/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/_CodeSignature/CodeResources b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/_CodeSignature/CodeResources new file mode 100644 index 00000000..1e94d0ff --- /dev/null +++ b/ios/Llama.xcframework/ios-arm64_x86_64-simulator/Llama.framework/_CodeSignature/CodeResources @@ -0,0 +1,223 @@ + + + + + files + + Headers/ggml-alloc.h + + I0o1yNqMayC5htF3onYJoQOrjpE= + + Headers/ggml-backend.h + + BXJOIu9ZRY0KkU+9j4gvnFF1lfs= + + Headers/ggml-cpu.h + + hO3RnG3fEsoqT2LnyHNIkD5ylQA= + + Headers/ggml-metal.h + + OPKWzK2BQxc8uydhDsIS9Ot2qCE= + + Headers/ggml-opt.h + + f9+8P3HT1FQTRJZGtBT0sEIv9VE= + + Headers/ggml.h + + mISflhL9UXaRyMWayFkDfFedOrc= + + Headers/gguf.h + + VLK4ktzJlOdm9759ZJ+Y74si958= + + Headers/llama.h + + P79ZX6ISVVhv6WNspP0FAw0T6lo= + + Headers/mtmd-helper.h + + zzc0rHtTY+FMBCdBGNeaHoKLNX4= + + Headers/mtmd.h + + DrL4/IcJRdtDmGKglKnPuh8Tjk0= + + Info.plist + + iEX8HYaevfNCkb+BVWvj2hW/zUQ= + + Modules/module.modulemap + + GLxr401eUfp6oByYq4y5zddmobY= + + + files2 + + Headers/ggml-alloc.h + + hash2 + + lOTNBpuTE7LOs12s7JAZgeC7R42LsxA1txJr4JGZjCM= + + + Headers/ggml-backend.h + + hash2 + + piDoFbQ6RMxy1fIWYpo6kZgDNbYbw362stCBM2jDcE8= + + + Headers/ggml-cpu.h + + hash2 + + Gq/pflduo4wNpXUX+ySSlV0LacGnmYQkgfwGnW2dKO8= + + + Headers/ggml-metal.h + + hash2 + + Mi82zTDz6eeq17W5vGMHgBL9C3cGrE4k9XIbYE89iYA= + + + Headers/ggml-opt.h + + hash2 + + NYbeG8ipNLXHIznitpN7BkHo8Um1EiMeZm9n3gc27qI= + + + Headers/ggml.h + + hash2 + + 7fzmpz5ifSIN2K68xtYgkXVbEAYUrX2lMpKlKLGwB1A= + + + Headers/gguf.h + + hash2 + + Ldsnalvs50NDMWCthjJ5RzQxydTBcdRov4YKPNo/vz8= + + + Headers/llama.h + + hash2 + + VrCntN56IKB+D6+KWox7hbhvo6E2c4XcmMr+Gm3HrUY= + + + Headers/mtmd-helper.h + + hash2 + + U+xT7AinBJ5PF493agD+pZ7wGPsLuw8sgAH17wfcCxk= + + + Headers/mtmd.h + + hash2 + + xX/4y6LQ5ZtrHg6Q/snhiHXMUmi65nNvxIuii2TPnPE= + + + Modules/module.modulemap + + hash2 + + YrZrr8fQjLcP/nkdW+LigUSHc1OnNtYwwRm5qRmrFFw= + + + + rules + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Headers b/ios/Llama.xcframework/macos-arm64/Llama.framework/Headers new file mode 120000 index 00000000..a177d2a6 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Headers @@ -0,0 +1 @@ +Versions/Current/Headers \ No newline at end of file diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Llama b/ios/Llama.xcframework/macos-arm64/Llama.framework/Llama new file mode 120000 index 00000000..40a539c6 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Llama @@ -0,0 +1 @@ +Versions/Current/Llama \ No newline at end of file diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Modules b/ios/Llama.xcframework/macos-arm64/Llama.framework/Modules new file mode 120000 index 00000000..5736f318 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Modules @@ -0,0 +1 @@ +Versions/Current/Modules \ No newline at end of file diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Resources b/ios/Llama.xcframework/macos-arm64/Llama.framework/Resources new file mode 120000 index 00000000..953ee36f --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Resources @@ -0,0 +1 @@ +Versions/Current/Resources \ No newline at end of file diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-alloc.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-alloc.h new file mode 100644 index 00000000..a7926a21 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-alloc.h @@ -0,0 +1,86 @@ +#pragma once + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; +typedef struct ggml_backend_buffer * ggml_backend_buffer_t; +typedef struct ggml_backend * ggml_backend_t; + +// Tensor allocator +struct ggml_tallocr { + ggml_backend_buffer_t buffer; + void * base; + size_t alignment; + size_t offset; +}; + +GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer); +GGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor); + +// Graph allocator +/* + Example usage: + ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + + // optional: create a worst-case graph and reserve the buffers to avoid reallocations + ggml_gallocr_reserve(galloc, build_graph(max_batch)); + + // allocate the graph + struct ggml_cgraph * graph = build_graph(batch); + ggml_gallocr_alloc_graph(galloc, graph); + + printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0)); + + // evaluate the graph + ggml_backend_graph_compute(backend, graph); +*/ + +// special tensor flags for use with the graph allocator: +// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses +// ggml_set_output(): output tensors are never freed and never overwritten + +typedef struct ggml_gallocr * ggml_gallocr_t; + +GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft); +GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs); +GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc); + +// pre-allocate buffers from a measure graph - does not allocate or modify the graph +// call with a worst-case graph to avoid buffer reallocations +// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed +// returns false if the buffer allocation failed +// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes +GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph); +GGML_API void ggml_gallocr_reserve_n_size( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids, + size_t * sizes); +GGML_API bool ggml_gallocr_reserve_n( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids); + +// automatic reallocation if the topology changes when using a single buffer +// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers) +GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph); + +GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id); + +// Utils +// Create a buffer and allocate all the tensors in a ggml_context +// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft +// ggml_backend_alloc_ctx_tensors_from_buft returns NULL on failure or if all tensors in ctx are already allocated or zero-sized +GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-backend.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-backend.h new file mode 100644 index 00000000..2924fdbe --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-backend.h @@ -0,0 +1,435 @@ +#pragma once + +#include "ggml.h" +#include "ggml-alloc.h" + +#ifdef GGML_BACKEND_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BACKEND_BUILD +# define GGML_BACKEND_API __declspec(dllexport) extern +# else +# define GGML_BACKEND_API __declspec(dllimport) extern +# endif +# else +# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_BACKEND_API extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; + typedef struct ggml_backend_buffer * ggml_backend_buffer_t; + typedef struct ggml_backend_event * ggml_backend_event_t; + typedef struct ggml_backend * ggml_backend_t; + typedef void * ggml_backend_graph_plan_t; + typedef struct ggml_backend_reg * ggml_backend_reg_t; + typedef struct ggml_backend_device * ggml_backend_dev_t; + + + // + // Backend buffer type + // + + GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); + GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); + GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft); + + // + // Backend buffer + // + + enum ggml_backend_buffer_usage { + GGML_BACKEND_BUFFER_USAGE_ANY = 0, + GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1, + GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2, + }; + + GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); + GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); + GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor); + GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); + GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer); + GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); + + // tensor copy between different backends + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); + + // + // Backend (stream) + // + + GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend); + GGML_API const char * ggml_backend_name(ggml_backend_t backend); + GGML_API void ggml_backend_free(ggml_backend_t backend); + + GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend); + GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size); + GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); + GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); + + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + + // "offset" refers to the offset in tensor->data for setting/getting data + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + + GGML_API void ggml_backend_synchronize(ggml_backend_t backend); + + GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + + GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); + + // NOTE: will be removed, use device version instead + GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); + GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op); + + // asynchronous copy + // the copy is performed after all the currently queued operations in backend_src + // backend_dst will wait for the copy to complete before performing other operations + // automatic fallback to sync copy if async is not supported + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); + + GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); + + // + // Events + // + + GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device); + GGML_API void ggml_backend_event_free(ggml_backend_event_t event); + GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend); + GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event); + GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event); + + // + // Backend device + // + + enum ggml_backend_dev_type { + // CPU device using system memory + GGML_BACKEND_DEVICE_TYPE_CPU, + // GPU device using dedicated memory + GGML_BACKEND_DEVICE_TYPE_GPU, + // integrated GPU device using host memory + GGML_BACKEND_DEVICE_TYPE_IGPU, + // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, + }; + + // functionality supported by the device + struct ggml_backend_dev_caps { + // asynchronous operations + bool async; + // pinned host buffer + bool host_buffer; + // creating buffers from host ptr + bool buffer_from_host_ptr; + // event synchronization + bool events; + }; + + // all the device properties + struct ggml_backend_dev_props { + // device name + const char * name; + // device description + const char * description; + // device free memory in bytes + size_t memory_free; + // device total memory in bytes + size_t memory_total; + // device type + enum ggml_backend_dev_type type; + // device id + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") + // if the id is unknown, this should be NULL + const char * device_id; + // device capabilities + struct ggml_backend_dev_caps caps; + }; + + GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device); + GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total); + GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props); + GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device); + GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size); + + GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + + // + // Backend (reg) + // + + GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg); + GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg); + GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index); + GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name); + + // Common functions that may be obtained using ggml_backend_reg_get_proc_address + + // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) + typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); + typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); + + // Split buffer type for tensor parallelism (old) + typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); + // Set the number of threads for the backend + typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); + // Get additional buffer types provided by the device (returns a NULL-terminated array) + typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device); + // Set the abort callback for the backend + typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data); + // Get a list of feature flags supported by the backend (returns a NULL-terminated array) + struct ggml_backend_feature { + const char * name; + const char * value; + }; + typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg); + + // + // Backend registry + // + + GGML_API void ggml_backend_register(ggml_backend_reg_t reg); + + GGML_API void ggml_backend_device_register(ggml_backend_dev_t device); + + // Backend (reg) enumeration + GGML_API size_t ggml_backend_reg_count(void); + GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index); + GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name); + + // Device enumeration + GGML_API size_t ggml_backend_dev_count(void); + GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type); + + // Direct backend (stream) initialization + // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params) + GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params) + GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL) + GGML_API ggml_backend_t ggml_backend_init_best(void); + + // Load a backend from a dynamic library and register it + GGML_API ggml_backend_reg_t ggml_backend_load(const char * path); + // Unload a backend if loaded dynamically and unregister it + GGML_API void ggml_backend_unload(ggml_backend_reg_t reg); + // Load all known backends from dynamic libraries + GGML_API void ggml_backend_load_all(void); + GGML_API void ggml_backend_load_all_from_path(const char * dir_path); + + // + // Backend scheduler + // + + // The backend scheduler allows for multiple backend devices to be used together + // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends + // The backends are selected based on: + // - the backend that supports the operation + // - the location of the pre-allocated tensors (e.g. the weights) + /* + Example usage: + + // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned + // preferably to run on the same backend as the buffer + ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true); + + // initialize buffers from a max size graph (optional) + reserve_graph = build_graph(sched, max_batch_size); + + // manually assign nodes to a backend (optional, should not be needed in most cases) + struct ggml_tensor * node = ggml_mul_mat(ctx, ...); + ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu); + + ggml_backend_sched_reserve(sched, reserve_graph); + + // compute + graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation + for (int i = 0; i < 10; ++i) { + ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically + } + + // if there are graph inputs: + graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called) + ggml_backend_sched_reset(sched); // clear the allocation of the previous graph + ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it + ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors + ggml_backend_sched_graph_compute(sched, graph); // execute the graph + + // as an alternative to the above it is also possible to assign the inputs to a dedicated context and + // allocate them statically via ggml_backend_alloc_ctx_tensors + } + */ + + typedef struct ggml_backend_sched * ggml_backend_sched_t; + + // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback) + // when ask == true, the scheduler wants to know if the user wants to observe this node + // this allows the scheduler to batch nodes together in order to evaluate them in a single call + // + // when ask == false, the scheduler is passing the node tensor to the user for observation + // if the user returns false, the scheduler will cancel the graph compute + // + typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); + + // Initialize a backend scheduler, backends with low index are given priority over backends with high index + GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload); + GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); + + // Initialize backend buffers from a measure graph + GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes); + GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success + + GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched); + GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i); + + // Get the number of splits of the last graph + GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); + GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched); + + GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend); + GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + + GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); + + // Split graph without allocating it + GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + + // Allocate and compute graph on the backend scheduler + GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success + GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched); + + // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph. + // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers. + // The correct way to use this API is to discard the deallocated tensors and create new ones. + GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); + + // Set a callback to be called for each resulting node during graph compute + GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + + // + // Meta backend + // + +#define GGML_BACKEND_META_MAX_DEVICES 16 + + enum ggml_backend_meta_split_axis { + // tensor split by tensor dimensions: + GGML_BACKEND_SPLIT_AXIS_0 = 0, + GGML_BACKEND_SPLIT_AXIS_1 = 1, + GGML_BACKEND_SPLIT_AXIS_2 = 2, + GGML_BACKEND_SPLIT_AXIS_3 = 3, + + GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends + GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum + + // for internal bookkeeping only: + GGML_BACKEND_SPLIT_AXIS_NONE = 98, + GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99, + }; + GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis); + + struct ggml_backend_meta_split_state { + enum ggml_backend_meta_split_axis axis; + + // for tensors with axis >= 0 && axis < GGML_MAX_DIMS: + // - each device has a slice of the tensor along the split axis + // - most tensors have n_segments == 1 and a contiguous slice of the tensor data + // - some tensors have an inhomogenenous data layout along the split axis, + // those tensors are divided into segments which are each individually split across devices + // - ne has one entry per segment and device and that segment repeats nr times, + // in total when accounting for repetitions the segments add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0_r0, seg0_dev1_r0, seg0_dev0_r1, seg0_dev1_r1, seg1_dev0_r0, seg1_dev1_r0], + // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V, + // the Q matrix can be larger than the K and V matrices so this can either be expressed as 3 segments or as 2 segments + // where the segment for K/V repeats twice + int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t nr[16]; + uint32_t n_segments; + }; + + // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible: + typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata); + + // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this: + // TODO: this looks a bit strange - a backend API creates a device. I think we should try + // express this as a backend registry functionality instead + GGML_API ggml_backend_dev_t ggml_backend_meta_device( + ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud); + + // + // Utils + // + + struct ggml_backend_graph_copy { + ggml_backend_buffer_t buffer; + struct ggml_context * ctx_allocated; + struct ggml_context * ctx_unallocated; + struct ggml_cgraph * graph; + }; + + // Copy a graph to a different backend + GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph); + GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy); + + typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data); + + // Compare the output of two backends + GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes); + + // Tensor initialization + GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr); + GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor); + + // CPU buffer types are always available + GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); + GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-cpu.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-cpu.h new file mode 100644 index 00000000..e3e067c9 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-cpu.h @@ -0,0 +1,151 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + + // the compute plan that needs to be prepared for ggml_graph_compute() + // since https://github.com/ggml-org/ggml/issues/287 + struct ggml_cplan { + size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()` + uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()` + + int n_threads; + struct ggml_threadpool * threadpool; + + // abort ggml_graph_compute when true + ggml_abort_callback abort_callback; + void * abort_callback_data; + + // use only reference implementations + bool use_ref; + }; + + // numa strategies + enum ggml_numa_strategy { + GGML_NUMA_STRATEGY_DISABLED = 0, + GGML_NUMA_STRATEGY_DISTRIBUTE = 1, + GGML_NUMA_STRATEGY_ISOLATE = 2, + GGML_NUMA_STRATEGY_NUMACTL = 3, + GGML_NUMA_STRATEGY_MIRROR = 4, + GGML_NUMA_STRATEGY_COUNT + }; + + GGML_BACKEND_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems + GGML_BACKEND_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node + + GGML_BACKEND_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value); + GGML_BACKEND_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value); + + GGML_BACKEND_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value); + GGML_BACKEND_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value); + + GGML_BACKEND_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i); + GGML_BACKEND_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value); + + GGML_BACKEND_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3); + GGML_BACKEND_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value); + + GGML_BACKEND_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i); + GGML_BACKEND_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value); + + GGML_BACKEND_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3); + GGML_BACKEND_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value); + + GGML_BACKEND_API struct ggml_threadpool * ggml_threadpool_new (struct ggml_threadpool_params * params); + GGML_BACKEND_API void ggml_threadpool_free (struct ggml_threadpool * threadpool); + GGML_BACKEND_API int ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool); + GGML_BACKEND_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool); + GGML_BACKEND_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool); + + // ggml_graph_plan() has to be called before ggml_graph_compute() + // when plan.work_size > 0, caller must allocate memory for plan.work_data + GGML_BACKEND_API struct ggml_cplan ggml_graph_plan( + const struct ggml_cgraph * cgraph, + int n_threads, /* = GGML_DEFAULT_N_THREADS */ + struct ggml_threadpool * threadpool /* = NULL */ ); + GGML_BACKEND_API enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan); + + // same as ggml_graph_compute() but the work data is allocated as a part of the context + // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data + GGML_BACKEND_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads); + + // + // system info + // + + // x86 + GGML_BACKEND_API int ggml_cpu_has_sse3 (void); + GGML_BACKEND_API int ggml_cpu_has_ssse3 (void); + GGML_BACKEND_API int ggml_cpu_has_avx (void); + GGML_BACKEND_API int ggml_cpu_has_avx_vnni (void); + GGML_BACKEND_API int ggml_cpu_has_avx2 (void); + GGML_BACKEND_API int ggml_cpu_has_bmi2 (void); + GGML_BACKEND_API int ggml_cpu_has_f16c (void); + GGML_BACKEND_API int ggml_cpu_has_fma (void); + GGML_BACKEND_API int ggml_cpu_has_avx512 (void); + GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void); + GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void); + GGML_BACKEND_API int ggml_cpu_has_avx512_bf16(void); + GGML_BACKEND_API int ggml_cpu_has_amx_int8 (void); + // ARM + GGML_BACKEND_API int ggml_cpu_has_neon (void); + GGML_BACKEND_API int ggml_cpu_has_arm_fma (void); + GGML_BACKEND_API int ggml_cpu_has_fp16_va (void); + GGML_BACKEND_API int ggml_cpu_has_dotprod (void); + GGML_BACKEND_API int ggml_cpu_has_matmul_int8(void); + GGML_BACKEND_API int ggml_cpu_has_sve (void); + GGML_BACKEND_API int ggml_cpu_get_sve_cnt (void); // sve vector length in bytes + GGML_BACKEND_API int ggml_cpu_has_sme (void); + // other + GGML_BACKEND_API int ggml_cpu_has_riscv_v (void); + GGML_BACKEND_API int ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytes + GGML_BACKEND_API int ggml_cpu_has_vsx (void); + GGML_BACKEND_API int ggml_cpu_has_vxe (void); + GGML_BACKEND_API int ggml_cpu_has_wasm_simd (void); + GGML_BACKEND_API int ggml_cpu_has_llamafile (void); + + // Internal types and functions exposed for tests and benchmarks + + typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx, + const void * GGML_RESTRICT y, size_t by, int nrc); + + struct ggml_type_traits_cpu { + ggml_from_float_t from_float; + ggml_vec_dot_t vec_dot; + enum ggml_type vec_dot_type; + int64_t nrows; // number of rows to process simultaneously + }; + + GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type); + + GGML_BACKEND_API void ggml_cpu_init(void); + + // + // CPU backend + // + + GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void); + + GGML_BACKEND_API bool ggml_backend_is_cpu (ggml_backend_t backend); + GGML_BACKEND_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads); + GGML_BACKEND_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool); + GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data); + + GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref); + + GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void); + + GGML_BACKEND_API void ggml_cpu_fp32_to_fp32(const float *, float *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_i32 (const float *, int32_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_fp16(const float *, ggml_fp16_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp16_to_fp32(const ggml_fp16_t *, float *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_bf16(const float *, ggml_bf16_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_bf16_to_fp32(const ggml_bf16_t *, float *, int64_t); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-metal.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-metal.h new file mode 100644 index 00000000..433838f0 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-metal.h @@ -0,0 +1,61 @@ +// Note: this description is outdated +// +// An interface allowing to compute ggml_cgraph with Metal +// +// This is a fully functional interface that extends ggml with GPU support for Apple devices. +// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.) +// +// How it works? +// +// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this +// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you +// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.) +// +// You only need to make sure that all memory buffers that you used during the graph creation +// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is +// used during the graph evaluation to determine the arguments of the compute kernels. +// +// Synchronization between device and host memory (for example for input and output tensors) +// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions. +// + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include + +struct ggml_tensor; +struct ggml_cgraph; + +#ifdef __cplusplus +extern "C" { +#endif + +// +// backend API +// user-code should use only these functions +// + +// TODO: remove in the future +GGML_BACKEND_API ggml_backend_t ggml_backend_metal_init(void); + +GGML_BACKEND_API bool ggml_backend_is_metal(ggml_backend_t backend); + +GGML_BACKEND_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data); + +// helper to check if the device supports a specific family +// ideally, the user code should be doing these checks +// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf +GGML_BACKEND_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family); + +// capture all command buffers committed the next time `ggml_backend_graph_compute` is called +GGML_BACKEND_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_metal_reg(void); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-opt.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-opt.h new file mode 100644 index 00000000..1c2ed79b --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml-opt.h @@ -0,0 +1,256 @@ +// This file contains functionality for training models using GGML. +// It is not strictly needed vs. just vanilla GGML but it provides a more high-level interface for common needs such as datasets. +// At the bottom of this file especially there are relatively high-level functions that are suitable use or adaptation in user code. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + struct ggml_opt_dataset; + struct ggml_opt_context; + struct ggml_opt_result; + + typedef struct ggml_opt_dataset * ggml_opt_dataset_t; + typedef struct ggml_opt_context * ggml_opt_context_t; + typedef struct ggml_opt_result * ggml_opt_result_t; + + // ====== Loss ====== + + // built-in loss types, i.e. the built-in quantities minimized by the optimizer + // custom loss types can be defined via mean or sum which simply reduce the outputs for all datapoints to a single value + enum ggml_opt_loss_type { + GGML_OPT_LOSS_TYPE_MEAN, + GGML_OPT_LOSS_TYPE_SUM, + GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, + GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR, + }; + + // ====== Dataset ====== + + GGML_API ggml_opt_dataset_t ggml_opt_dataset_init( + enum ggml_type type_data, // the type for the internal data tensor + enum ggml_type type_label, // the type for the internal labels tensor + int64_t ne_datapoint, // number of elements per datapoint + int64_t ne_label, // number of elements per label + int64_t ndata, // total number of datapoints/labels + int64_t ndata_shard); // number of datapoints/labels per shard (unit at which the dataset is shuffled/copied) + GGML_API void ggml_opt_dataset_free(ggml_opt_dataset_t dataset); + + // get underlying tensors that store the data + GGML_API int64_t ggml_opt_dataset_ndata (ggml_opt_dataset_t dataset); + GGML_API struct ggml_tensor * ggml_opt_dataset_data (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata] + GGML_API struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset); // shape = [nd_label, ndata] + + // shuffle idata first datapoints from dataset with RNG from opt_ctx, shuffle all datapoints if idata is negative + GGML_API void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata); + + // get batch at position ibatch from dataset and copy the data to data_batch and labels_batch + GGML_API void ggml_opt_dataset_get_batch( + ggml_opt_dataset_t dataset, + struct ggml_tensor * data_batch, // shape = [ne_datapoint, ndata_batch] + struct ggml_tensor * labels_batch, // shape = [ne_label, ndata_batch] + int64_t ibatch); + GGML_API void ggml_opt_dataset_get_batch_host( + ggml_opt_dataset_t dataset, + void * data_batch, + size_t nb_data_batch, + void * labels_batch, + int64_t ibatch); + + // ====== Model / Context ====== + + enum ggml_opt_build_type { + GGML_OPT_BUILD_TYPE_FORWARD = 10, + GGML_OPT_BUILD_TYPE_GRAD = 20, + GGML_OPT_BUILD_TYPE_OPT = 30, + }; + + enum ggml_opt_optimizer_type { + GGML_OPT_OPTIMIZER_TYPE_ADAMW, + GGML_OPT_OPTIMIZER_TYPE_SGD, + + GGML_OPT_OPTIMIZER_TYPE_COUNT + }; + + // parameters that control which optimizer is used and how said optimizer tries to find the minimal loss + struct ggml_opt_optimizer_params { + struct { + float alpha; // learning rate + float beta1; // first AdamW momentum + float beta2; // second AdamW momentum + float eps; // epsilon for numerical stability + float wd; // weight decay - 0.0f to disable + } adamw; + struct { + float alpha; // learning rate + float wd; // weight decay + } sgd; + }; + + // callback to calculate optimizer parameters prior to a backward pass + // userdata can be used to pass arbitrary data + typedef struct ggml_opt_optimizer_params (*ggml_opt_get_optimizer_params)(void * userdata); + + // returns the default optimizer params (constant, hard-coded values) + // userdata is not used + GGML_API struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata); + + // casts userdata to ggml_opt_optimizer_params and returns it + GGML_API struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata); + + // parameters for initializing a new optimization context + struct ggml_opt_params { + ggml_backend_sched_t backend_sched; // defines which backends are used to construct the compute graphs + + // by default the forward graph needs to be reconstructed for each eval + // if ctx_compute, inputs, and outputs are set the graphs are instead allocated statically + struct ggml_context * ctx_compute; + struct ggml_tensor * inputs; + struct ggml_tensor * outputs; + + enum ggml_opt_loss_type loss_type; + enum ggml_opt_build_type build_type; + + int32_t opt_period; // after how many gradient accumulation steps an optimizer step should be done + + ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters + void * get_opt_pars_ud; // userdata for calculating optimizer parameters + + // only GGML_OPT_OPTIMIZER_TYPE_ADAMW needs m, v momenta per parameter tensor + enum ggml_opt_optimizer_type optimizer; + }; + + // get parameters for an optimization context with defaults set where possible + // parameters for which no sensible defaults exist are supplied as arguments to this function + GGML_API struct ggml_opt_params ggml_opt_default_params( + ggml_backend_sched_t backend_sched, + enum ggml_opt_loss_type loss_type); + + GGML_API ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params); + GGML_API void ggml_opt_free(ggml_opt_context_t opt_ctx); + + // set gradients to zero, initialize loss, and optionally reset the optimizer + GGML_API void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer); + + GGML_API bool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx); // whether the graphs are allocated_statically + + // get underlying tensors that store data + // if not using static graphs these pointers become invalid with the next call to ggml_opt_alloc + GGML_API struct ggml_tensor * ggml_opt_inputs( ggml_opt_context_t opt_ctx); // forward graph input tensor + GGML_API struct ggml_tensor * ggml_opt_outputs( ggml_opt_context_t opt_ctx); // forward graph output tensor + GGML_API struct ggml_tensor * ggml_opt_labels( ggml_opt_context_t opt_ctx); // labels to compare outputs against + GGML_API struct ggml_tensor * ggml_opt_loss( ggml_opt_context_t opt_ctx); // scalar tensor that contains the loss + GGML_API struct ggml_tensor * ggml_opt_pred( ggml_opt_context_t opt_ctx); // predictions made by outputs + GGML_API struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx); // number of matching predictions between outputs and labels + + // get the gradient accumulator for a node from the forward graph + GGML_API struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node); + + GGML_API enum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t); //TODO consistent naming scheme + + GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type); + + // ====== Optimization Result ====== + + GGML_API ggml_opt_result_t ggml_opt_result_init(void); + GGML_API void ggml_opt_result_free(ggml_opt_result_t result); + GGML_API void ggml_opt_result_reset(ggml_opt_result_t result); + + // get data from result, uncertainties are optional and can be ignored by passing NULL + GGML_API void ggml_opt_result_ndata( ggml_opt_result_t result, int64_t * ndata); // writes 1 value, number of datapoints + GGML_API void ggml_opt_result_loss( ggml_opt_result_t result, double * loss, double * unc); // writes 1 value + GGML_API void ggml_opt_result_pred( ggml_opt_result_t result, int32_t * pred); // writes ndata values + GGML_API void ggml_opt_result_accuracy(ggml_opt_result_t result, double * accuracy, double * unc); // writes 1 value + + // ====== Computation ====== + + // if not using static graphs, this function must be called prior to ggml_opt_alloc + GGML_API void ggml_opt_prepare_alloc( + ggml_opt_context_t opt_ctx, + struct ggml_context * ctx_compute, + struct ggml_cgraph * gf, + struct ggml_tensor * inputs, + struct ggml_tensor * outputs); + + // allocate the next graph for evaluation, either forward or forward + backward + // must be called exactly once prior to calling ggml_opt_eval + GGML_API void ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward); + + // do forward pass, increment result if not NULL, do backward pass if allocated + GGML_API void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result); + + // ############################################################################ + // ## The high-level functions start here. They do not depend on any private ## + // ## functions or structs and can be copied to and adapted for user code. ## + // ############################################################################ + + // ====== Intended Usage ====== + // + // 1. Select the appropriate loss for your problem. + // 2. Create a dataset and set the data for the "data" tensor. Also set the "labels" tensor if your loss needs them. + // Setting the shard size to 1 will be fine, it's the granularity with which data is shuffled/loaded (bigger values are faster). + // 3. Create a GGML graph for your model with no_alloc == true. Use two separate contexts for the tensors. + // The first context should contain the model parameters and inputs and be allocated statically in user code. + // The second context should contain all other tensors and will be (re)allocated automatically. + // Due to this automated allocation the data of the second context is not defined when accessed in user code. + // Note that the second dimension of the inputs/outputs are interpreted as the number of datapoints in those tensors. + // 4. Call ggml_opt_fit. If you need more control you can use ggml_opt_epoch instead. + + // signature for a callback while evaluating opt_ctx on dataset, called after an evaluation + typedef void (*ggml_opt_epoch_callback)( + bool train, // true after training evaluation, false after validation evaluation + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, // result associated with the dataset subsection + int64_t ibatch, // number of batches that have been evaluated so far + int64_t ibatch_max, // total number of batches in this dataset subsection + int64_t t_start_us); // time at which the evaluation on the dataset subsection was started + + // do training on front of dataset, do evaluation only on back of dataset + GGML_API void ggml_opt_epoch( + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, // result to increment during training, ignored if NULL + ggml_opt_result_t result_eval, // result to increment during evaluation, ignored if NULL + int64_t idata_split, // data index at which to split training and evaluation + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval); + + // callback that prints a progress bar on stderr + GGML_API void ggml_opt_epoch_callback_progress_bar( + bool train, + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, + int64_t ibatch, + int64_t ibatch_max, + int64_t t_start_us); + + // fit model defined by inputs and outputs to dataset + GGML_API void ggml_opt_fit( + ggml_backend_sched_t backend_sched, // backend scheduler for constructing the compute graphs + struct ggml_context * ctx_compute, // context with temporarily allocated tensors to calculate the outputs + struct ggml_tensor * inputs, // input tensor with shape [ne_datapoint, ndata_batch] + struct ggml_tensor * outputs, // output tensor, must have shape [ne_label, ndata_batch] if labels are used + ggml_opt_dataset_t dataset, // dataset with data and optionally also labels + enum ggml_opt_loss_type loss_type, // loss to minimize + enum ggml_opt_optimizer_type optimizer, // sgd or adamw + ggml_opt_get_optimizer_params get_opt_pars, // callback to get optimizer params, userdata is pointer to epoch (of type int64_t) + int64_t nepoch, // how many times the dataset should be iterated over + int64_t nbatch_logical, // datapoints optimizer step, must be a multiple of ndata_batch in inputs/outputs + float val_split, // fraction of the dataset to use for validation, must be in [0.0f, 1.0f) + bool silent); // whether or not info prints to stderr should be suppressed + + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml.h new file mode 100644 index 00000000..374934aa --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/ggml.h @@ -0,0 +1,2856 @@ +#pragma once + +// +// GGML Tensor Library +// +// This documentation is still a work in progress. +// If you wish some specific topics to be covered, feel free to drop a comment: +// +// https://github.com/ggml-org/whisper.cpp/issues/40 +// +// ## Overview +// +// This library implements: +// +// - a set of tensor operations +// - automatic differentiation +// - basic optimization algorithms +// +// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes, +// but is not limited to, the following: +// +// - linear regression +// - support vector machines +// - neural networks +// +// The library allows the user to define a certain function using the available tensor operations. This function +// definition is represented internally via a computation graph. Each tensor operation in the function definition +// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the +// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized +// using one of the available optimization algorithms. +// +// For example, here we define the function: f(x) = a*x^2 + b +// +// { +// struct ggml_init_params params = { +// .mem_size = 16*1024*1024, +// .mem_buffer = NULL, +// }; +// +// // memory allocation happens here +// struct ggml_context * ctx = ggml_init(params); +// +// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// +// ggml_set_param(ctx, x); // x is an input variable +// +// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * x2 = ggml_mul(ctx, x, x); +// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b); +// +// ... +// } +// +// Notice that the function definition above does not involve any actual computation. The computation is performed only +// when the user explicitly requests it. For example, to compute the function's value at x = 2.0: +// +// { +// ... +// +// struct ggml_cgraph * gf = ggml_new_graph(ctx); +// ggml_build_forward_expand(gf, f); +// +// // set the input variable and parameter values +// ggml_set_f32(x, 2.0f); +// ggml_set_f32(a, 3.0f); +// ggml_set_f32(b, 4.0f); +// +// ggml_graph_compute_with_ctx(ctx, &gf, n_threads); +// +// printf("f = %f\n", ggml_get_f32_1d(f, 0)); +// +// ... +// } +// +// The actual computation is performed in the ggml_graph_compute() function. +// +// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the +// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know +// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory +// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was +// actually needed. +// +// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic +// differentiation and optimization algorithms. +// +// The described approach allows to define the function graph once and then compute its forward or backward graphs +// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way +// the user can avoid the memory allocation overhead at runtime. +// +// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class +// citizens, but in theory the library can be extended to support FP8 and integer data types. +// +// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary +// and binary operations. Most of the available operations fall into one of these two categories. With time, it became +// clear that the library needs to support more complex operations. The way to support these operations is not clear +// yet, but a few examples are demonstrated in the following operations: +// +// - ggml_permute() +// - ggml_conv_1d_1s() +// - ggml_conv_1d_2s() +// +// For each tensor operator, the library implements a forward and backward computation function. The forward function +// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the +// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a +// calculus class, or watch the following video: +// +// What is Automatic Differentiation? +// https://www.youtube.com/watch?v=wG_nF1awSSY +// +// +// ## Tensor data (struct ggml_tensor) +// +// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of +// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains +// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example: +// +// { +// struct ggml_tensor * c = ggml_add(ctx, a, b); +// +// assert(c->src[0] == a); +// assert(c->src[1] == b); +// } +// +// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the +// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows +// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and +// permutation. All tensor operations have to take the stride into account and not assume that the tensor is +// contiguous in memory. +// +// The data of the tensor is accessed via the "data" pointer. For example: +// +// { +// const int nx = 2; +// const int ny = 3; +// +// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny); +// +// for (int y = 0; y < ny; y++) { +// for (int x = 0; x < nx; x++) { +// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y; +// } +// } +// +// ... +// } +// +// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used. +// +// ## The matrix multiplication operator (ggml_mul_mat) +// +// TODO +// +// +// ## Multi-threading +// +// TODO +// +// +// ## Overview of ggml.c +// +// TODO +// +// +// ## SIMD optimizations +// +// TODO +// +// +// ## Debugging ggml +// +// TODO +// +// + +#ifdef GGML_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BUILD +# define GGML_API __declspec(dllexport) extern +# else +# define GGML_API __declspec(dllimport) extern +# endif +# else +# define GGML_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_API extern +#endif + +// TODO: support for clang +#ifdef __GNUC__ +# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define GGML_DEPRECATED(func, hint) func +#endif + +#ifndef __GNUC__ +# define GGML_ATTRIBUTE_FORMAT(...) +#elif defined(__MINGW32__) && !defined(__clang__) +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__))) +#else +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__))) +#endif + +#if defined(_WIN32) && !defined(_WIN32_WINNT) +# define _WIN32_WINNT 0x0A00 +#endif + +#include +#include +#include +#include + +#define GGML_FILE_MAGIC 0x67676d6c // "ggml" +#define GGML_FILE_VERSION 2 + +#define GGML_QNT_VERSION 2 // bump this on quantization format changes +#define GGML_QNT_VERSION_FACTOR 1000 // do not change this + +#define GGML_MAX_DIMS 4 +#define GGML_MAX_PARAMS 2048 +#define GGML_MAX_SRC 10 +#define GGML_MAX_N_THREADS 512 +#define GGML_MAX_OP_PARAMS 64 + +#ifndef GGML_MAX_NAME +# define GGML_MAX_NAME 64 +#endif + +#define GGML_DEFAULT_N_THREADS 4 +#define GGML_DEFAULT_GRAPH_SIZE 2048 + +#if UINTPTR_MAX == 0xFFFFFFFF + #define GGML_MEM_ALIGN 4 +#elif defined(__EMSCRIPTEN__) +// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm. +// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.) +// ref: https://github.com/ggml-org/llama.cpp/pull/18628 + #define GGML_MEM_ALIGN 8 +#else + #define GGML_MEM_ALIGN 16 +#endif + +#define GGML_EXIT_SUCCESS 0 +#define GGML_EXIT_ABORTED 1 + +// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726 +#define GGML_ROPE_TYPE_NORMAL 0 +#define GGML_ROPE_TYPE_NEOX 2 +#define GGML_ROPE_TYPE_MROPE 8 +#define GGML_ROPE_TYPE_VISION 24 +#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000 + +#define GGML_MROPE_SECTIONS 4 + +#define GGML_UNUSED(x) (void)(x) +#ifdef __CUDACC__ +template +__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {} +#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__) +#else +#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0) +#endif // __CUDACC__ + +#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) + +#ifndef NDEBUG +# define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0) +#elif defined(__GNUC__) +# define GGML_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) +# define GGML_UNREACHABLE() __assume(0) +#else +# define GGML_UNREACHABLE() ((void) 0) +#endif + +#ifdef __cplusplus +# define GGML_NORETURN [[noreturn]] +#elif defined(_MSC_VER) +# define GGML_NORETURN __declspec(noreturn) +#else +# define GGML_NORETURN _Noreturn +#endif + +#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__) +#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x) + +// used to copy the number of elements and stride in bytes of tensors into local variables. +// main purpose is to reduce code duplication and improve readability. +// +// example: +// +// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); +// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb); +// +#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \ + const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \ + GGML_UNUSED(prefix##0); +#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \ + const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \ + GGML_UNUSED(prefix##1); +#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \ + const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \ + GGML_UNUSED(prefix##2); +#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \ + const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \ + GGML_UNUSED(prefix##3); + +#define GGML_TENSOR_UNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_TERNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \ + GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS01 \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) + +#ifdef __cplusplus +extern "C" { +#endif + + // Function type used in fatal error callbacks + typedef void (*ggml_abort_callback_t)(const char * error_message); + + // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout) + // Returns the old callback for chaining + GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback); + + GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4) + GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...); + + enum ggml_status { + GGML_STATUS_ALLOC_FAILED = -2, + GGML_STATUS_FAILED = -1, + GGML_STATUS_SUCCESS = 0, + GGML_STATUS_ABORTED = 1, + }; + + // get ggml_status name string + GGML_API const char * ggml_status_to_string(enum ggml_status status); + + // ieee 754-2008 half-precision float16 + // todo: make this not an integral type + typedef uint16_t ggml_fp16_t; + GGML_API float ggml_fp16_to_fp32(ggml_fp16_t); + GGML_API ggml_fp16_t ggml_fp32_to_fp16(float); + GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t); + + // google brain half-precision bfloat16 + typedef struct { uint16_t bits; } ggml_bf16_t; + GGML_API ggml_bf16_t ggml_fp32_to_bf16(float); + GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16 + GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t); + GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t); + + struct ggml_object; + struct ggml_context; + struct ggml_cgraph; + + // NOTE: always add types at the end of the enum to keep backward compatibility + enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + // GGML_TYPE_Q4_2 = 4, support has been removed + // GGML_TYPE_Q4_3 = 5, support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, + GGML_TYPE_IQ2_XXS = 16, + GGML_TYPE_IQ2_XS = 17, + GGML_TYPE_IQ3_XXS = 18, + GGML_TYPE_IQ1_S = 19, + GGML_TYPE_IQ4_NL = 20, + GGML_TYPE_IQ3_S = 21, + GGML_TYPE_IQ2_S = 22, + GGML_TYPE_IQ4_XS = 23, + GGML_TYPE_I8 = 24, + GGML_TYPE_I16 = 25, + GGML_TYPE_I32 = 26, + GGML_TYPE_I64 = 27, + GGML_TYPE_F64 = 28, + GGML_TYPE_IQ1_M = 29, + GGML_TYPE_BF16 = 30, + // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // GGML_TYPE_Q4_0_4_8 = 32, + // GGML_TYPE_Q4_0_8_8 = 33, + GGML_TYPE_TQ1_0 = 34, + GGML_TYPE_TQ2_0 = 35, + // GGML_TYPE_IQ4_NL_4_4 = 36, + // GGML_TYPE_IQ4_NL_4_8 = 37, + // GGML_TYPE_IQ4_NL_8_8 = 38, + GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) + GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_COUNT = 42, + }; + + // precision + enum ggml_prec { + GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default + GGML_PREC_F32 = 10, + }; + + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, + }; + + // model file types + enum ggml_ftype { + GGML_FTYPE_UNKNOWN = -1, + GGML_FTYPE_ALL_F32 = 0, + GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors + GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors + GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors + GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors + GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + }; + + // available tensor operations: + enum ggml_op { + GGML_OP_NONE = 0, + + GGML_OP_DUP, + GGML_OP_ADD, + GGML_OP_ADD_ID, + GGML_OP_ADD1, + GGML_OP_ACC, + GGML_OP_SUB, + GGML_OP_MUL, + GGML_OP_DIV, + GGML_OP_SQR, + GGML_OP_SQRT, + GGML_OP_LOG, + GGML_OP_SIN, + GGML_OP_COS, + GGML_OP_SUM, + GGML_OP_SUM_ROWS, + GGML_OP_CUMSUM, + GGML_OP_MEAN, + GGML_OP_ARGMAX, + GGML_OP_COUNT_EQUAL, + GGML_OP_REPEAT, + GGML_OP_REPEAT_BACK, + GGML_OP_CONCAT, + GGML_OP_SILU_BACK, + GGML_OP_NORM, // normalize + GGML_OP_RMS_NORM, + GGML_OP_RMS_NORM_BACK, + GGML_OP_GROUP_NORM, + GGML_OP_L2_NORM, + + GGML_OP_MUL_MAT, + GGML_OP_MUL_MAT_ID, + GGML_OP_OUT_PROD, + + GGML_OP_SCALE, + GGML_OP_SET, + GGML_OP_CPY, + GGML_OP_CONT, + GGML_OP_RESHAPE, + GGML_OP_VIEW, + GGML_OP_PERMUTE, + GGML_OP_TRANSPOSE, + GGML_OP_GET_ROWS, + GGML_OP_GET_ROWS_BACK, + GGML_OP_SET_ROWS, + GGML_OP_DIAG, + GGML_OP_DIAG_MASK_INF, + GGML_OP_DIAG_MASK_ZERO, + GGML_OP_SOFT_MAX, + GGML_OP_SOFT_MAX_BACK, + GGML_OP_ROPE, + GGML_OP_ROPE_BACK, + GGML_OP_CLAMP, + GGML_OP_CONV_TRANSPOSE_1D, + GGML_OP_IM2COL, + GGML_OP_IM2COL_BACK, + GGML_OP_IM2COL_3D, + GGML_OP_COL2IM_1D, + GGML_OP_CONV_2D, + GGML_OP_CONV_3D, + GGML_OP_CONV_2D_DW, + GGML_OP_CONV_TRANSPOSE_2D, + GGML_OP_POOL_1D, + GGML_OP_POOL_2D, + GGML_OP_POOL_2D_BACK, + GGML_OP_UPSCALE, + GGML_OP_PAD, + GGML_OP_PAD_REFLECT_1D, + GGML_OP_ROLL, + GGML_OP_ARANGE, + GGML_OP_TIMESTEP_EMBEDDING, + GGML_OP_ARGSORT, + GGML_OP_TOP_K, + GGML_OP_LEAKY_RELU, + GGML_OP_TRI, + GGML_OP_FILL, + + GGML_OP_FLASH_ATTN_EXT, + GGML_OP_FLASH_ATTN_BACK, + GGML_OP_SSM_CONV, + GGML_OP_SSM_SCAN, + GGML_OP_WIN_PART, + GGML_OP_WIN_UNPART, + GGML_OP_GET_REL_POS, + GGML_OP_ADD_REL_POS, + GGML_OP_RWKV_WKV6, + GGML_OP_GATED_LINEAR_ATTN, + GGML_OP_RWKV_WKV7, + GGML_OP_SOLVE_TRI, + GGML_OP_GATED_DELTA_NET, + + GGML_OP_UNARY, + + GGML_OP_MAP_CUSTOM1, + GGML_OP_MAP_CUSTOM2, + GGML_OP_MAP_CUSTOM3, + + GGML_OP_CUSTOM, + + GGML_OP_CROSS_ENTROPY_LOSS, + GGML_OP_CROSS_ENTROPY_LOSS_BACK, + GGML_OP_OPT_STEP_ADAMW, + GGML_OP_OPT_STEP_SGD, + + GGML_OP_GLU, + + GGML_OP_COUNT, + }; + + enum ggml_unary_op { + GGML_UNARY_OP_ABS, + GGML_UNARY_OP_SGN, + GGML_UNARY_OP_NEG, + GGML_UNARY_OP_STEP, + GGML_UNARY_OP_TANH, + GGML_UNARY_OP_ELU, + GGML_UNARY_OP_RELU, + GGML_UNARY_OP_SIGMOID, + GGML_UNARY_OP_GELU, + GGML_UNARY_OP_GELU_QUICK, + GGML_UNARY_OP_SILU, + GGML_UNARY_OP_HARDSWISH, + GGML_UNARY_OP_HARDSIGMOID, + GGML_UNARY_OP_EXP, + GGML_UNARY_OP_EXPM1, + GGML_UNARY_OP_SOFTPLUS, + GGML_UNARY_OP_GELU_ERF, + GGML_UNARY_OP_XIELU, + GGML_UNARY_OP_FLOOR, + GGML_UNARY_OP_CEIL, + GGML_UNARY_OP_ROUND, + GGML_UNARY_OP_TRUNC, + + GGML_UNARY_OP_COUNT, + }; + + enum ggml_glu_op { + GGML_GLU_OP_REGLU, + GGML_GLU_OP_GEGLU, + GGML_GLU_OP_SWIGLU, + GGML_GLU_OP_SWIGLU_OAI, + GGML_GLU_OP_GEGLU_ERF, + GGML_GLU_OP_GEGLU_QUICK, + + GGML_GLU_OP_COUNT, + }; + + enum ggml_object_type { + GGML_OBJECT_TYPE_TENSOR, + GGML_OBJECT_TYPE_GRAPH, + GGML_OBJECT_TYPE_WORK_BUFFER + }; + + enum ggml_log_level { + GGML_LOG_LEVEL_NONE = 0, + GGML_LOG_LEVEL_DEBUG = 1, + GGML_LOG_LEVEL_INFO = 2, + GGML_LOG_LEVEL_WARN = 3, + GGML_LOG_LEVEL_ERROR = 4, + GGML_LOG_LEVEL_CONT = 5, // continue previous log + }; + + // this tensor... + enum ggml_tensor_flag { + GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph + GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph + GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters + GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up) + GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed + }; + + enum ggml_tri_type { + GGML_TRI_TYPE_UPPER_DIAG = 0, + GGML_TRI_TYPE_UPPER = 1, + GGML_TRI_TYPE_LOWER_DIAG = 2, + GGML_TRI_TYPE_LOWER = 3 + }; + + struct ggml_init_params { + // memory pool + size_t mem_size; // bytes + void * mem_buffer; // if NULL, memory will be allocated internally + bool no_alloc; // don't allocate memory for the tensor data + }; + + // n-dimensional tensor + struct ggml_tensor { + enum ggml_type type; + + struct ggml_backend_buffer * buffer; + + int64_t ne[GGML_MAX_DIMS]; // number of elements + size_t nb[GGML_MAX_DIMS]; // stride in bytes: + // nb[0] = ggml_type_size(type) + // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding + // nb[i] = nb[i-1] * ne[i-1] + + // compute data + enum ggml_op op; + + // op params - allocated as int32_t for alignment + int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; + + int32_t flags; + + struct ggml_tensor * src[GGML_MAX_SRC]; + + // source tensor and offset for views + struct ggml_tensor * view_src; + size_t view_offs; + + void * data; + + char name[GGML_MAX_NAME]; + + void * extra; // extra things e.g. for ggml-cuda.cu + + char padding[8]; + }; + + static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); + + // Abort callback + // If not NULL, called before ggml computation + // If it returns true, the computation is aborted + typedef bool (*ggml_abort_callback)(void * data); + + + // + // GUID + // + + // GUID types + typedef uint8_t ggml_guid[16]; + typedef ggml_guid * ggml_guid_t; + + GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b); + + // misc + + GGML_API const char * ggml_version(void); + GGML_API const char * ggml_commit(void); + + GGML_API void ggml_time_init(void); // call this once at the beginning of the program + GGML_API int64_t ggml_time_ms(void); + GGML_API int64_t ggml_time_us(void); + GGML_API int64_t ggml_cycles(void); + GGML_API int64_t ggml_cycles_per_ms(void); + + // accepts a UTF-8 path, even on Windows + GGML_API FILE * ggml_fopen(const char * fname, const char * mode); + + GGML_API void ggml_print_object (const struct ggml_object * obj); + GGML_API void ggml_print_objects(const struct ggml_context * ctx); + + GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor); + GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN + + GGML_API int64_t ggml_blck_size(enum ggml_type type); + GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block + GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row + + GGML_DEPRECATED( + GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float + "use ggml_row_size() instead"); + + GGML_API const char * ggml_type_name(enum ggml_type type); + GGML_API const char * ggml_op_name (enum ggml_op op); + GGML_API const char * ggml_op_symbol(enum ggml_op op); + + GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op); + GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op); + GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name + + GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor); + + GGML_API bool ggml_is_quantized(enum ggml_type type); + + // TODO: temporary until model loading of ggml examples is refactored + GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype); + + GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor); + GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_view (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor); + GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars + + // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation) + GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous() + GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1 + GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2 + + // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok) + GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor); + + // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN + GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor); + + // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements + GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor); + + GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1); + GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + // use this to compute the memory overhead of a tensor + GGML_API size_t ggml_tensor_overhead(void); + + GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes); + + // main + + GGML_API struct ggml_context * ggml_init (struct ggml_init_params params); + GGML_API void ggml_reset(struct ggml_context * ctx); + GGML_API void ggml_free (struct ggml_context * ctx); + + GGML_API size_t ggml_used_mem(const struct ggml_context * ctx); + + GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx); + GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc); + + GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx); + GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx); + GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx); + + GGML_API struct ggml_tensor * ggml_new_tensor( + struct ggml_context * ctx, + enum ggml_type type, + int n_dims, + const int64_t *ne); + + GGML_API struct ggml_tensor * ggml_new_tensor_1d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_new_tensor_2d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_new_tensor_3d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_new_tensor_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes); + + GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src); + GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src); + + // Context tensor enumeration and lookup + GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx); + GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name); + + // Converts a flat index into coordinates + GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3); + + GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor); + GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor); + + GGML_API void * ggml_get_data (const struct ggml_tensor * tensor); + GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor); + + GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name); + GGML_ATTRIBUTE_FORMAT(2, 3) + GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...); + + // Tensor flags + GGML_API void ggml_set_input(struct ggml_tensor * tensor); + GGML_API void ggml_set_output(struct ggml_tensor * tensor); + GGML_API void ggml_set_param(struct ggml_tensor * tensor); + GGML_API void ggml_set_loss(struct ggml_tensor * tensor); + + // + // operations on tensors with backpropagation + // + + GGML_API struct ggml_tensor * ggml_dup( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_dup_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_add( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_type type); + + // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]] + GGML_API struct ggml_tensor * ggml_add_id( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add_inplace instead"); + + // dst = a + // view(dst, nb1, nb2, nb3, offset) += b + // return dst + GGML_API struct ggml_tensor * ggml_acc( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_acc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_sub( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sub_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sqr( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqr_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // return scalar + GGML_API struct ggml_tensor * ggml_sum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d] + GGML_API struct ggml_tensor * ggml_sum_rows( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cumsum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // mean along rows + GGML_API struct ggml_tensor * ggml_mean( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // argmax along rows + GGML_API struct ggml_tensor * ggml_argmax( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // count number of equal elements in a and b + GGML_API struct ggml_tensor * ggml_count_equal( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // if a is the same shape as b, and a is not parameter, return a + // otherwise, return a new tensor: repeat(a) to fit in b + GGML_API struct ggml_tensor * ggml_repeat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // repeat a to the specified shape + GGML_API struct ggml_tensor * ggml_repeat_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // sums repetitions in a into shape of b + GGML_API struct ggml_tensor * ggml_repeat_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride + + // concat a and b along dim + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_concat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int dim); + + GGML_API struct ggml_tensor * ggml_abs( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_abs_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_relu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_leaky_relu( + struct ggml_context * ctx, + struct ggml_tensor * a, float negative_slope, bool inplace); + + GGML_API struct ggml_tensor * ggml_relu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // GELU using erf (error function) when possible + // some backends may fallback to approximation based on Abramowitz and Stegun formula + GGML_API struct ggml_tensor * ggml_gelu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_erf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a - dy + // b - x + GGML_API struct ggml_tensor * ggml_silu_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // hardswish(x) = x * relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardswish( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // hardsigmoid(x) = relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardsigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + /** + * Truncates the fractional part of each element in the tensor (towards zero). + * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0 + * Similar to std::trunc in C/C++. + */ + + GGML_API struct ggml_tensor * ggml_trunc( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_trunc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + + + // xIELU activation function + // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0) + // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions + // that constrain the positive and negative source alpha values respectively + GGML_API struct ggml_tensor * ggml_xielu( + struct ggml_context * ctx, + struct ggml_tensor * a, + float alpha_n, + float alpha_p, + float beta, + float eps); + + // gated linear unit ops + // A: n columns, r rows, + // result is n / 2 columns, r rows, + // expects gate in second half of row, unless swapped is true + GGML_API struct ggml_tensor * ggml_glu( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_glu_op op, + bool swapped); + + GGML_API struct ggml_tensor * ggml_reglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_reglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // A: n columns, r rows, + // B: n columns, r rows, + GGML_API struct ggml_tensor * ggml_glu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_glu_op op); + + GGML_API struct ggml_tensor * ggml_reglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_erf_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_quick_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_oai( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float alpha, + float limit); + + // normalize along rows + GGML_API struct ggml_tensor * ggml_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // group normalize along ne0*ne1*n_groups + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_group_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + GGML_API struct ggml_tensor * ggml_group_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + // l2 normalize along rows + // used in rwkv v7 + GGML_API struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // a - x + // b - dy + GGML_API struct ggml_tensor * ggml_rms_norm_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float eps); + + // A: k columns, n rows => [ne03, ne02, n, k] + // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] + // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] + GGML_API struct ggml_tensor * ggml_mul_mat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // change the precision of a matrix multiplication + // set to GGML_PREC_F32 for higher precision (useful for phi-2) + GGML_API void ggml_mul_mat_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); + + // indirect matrix multiplication + GGML_API struct ggml_tensor * ggml_mul_mat_id( + struct ggml_context * ctx, + struct ggml_tensor * as, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + // A: m columns, n rows, + // B: p columns, n rows, + // result is m columns, p rows + GGML_API struct ggml_tensor * ggml_out_prod( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // + // operations on tensors without backpropagation + // + + GGML_API struct ggml_tensor * ggml_scale( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_scale_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // x = s * a + b + GGML_API struct ggml_tensor * ggml_scale_bias( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + GGML_API struct ggml_tensor * ggml_scale_bias_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_2d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // a -> b, return view(b) + GGML_API struct ggml_tensor * ggml_cpy( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // note: casting from f32 to i32 will discard the fractional part + GGML_API struct ggml_tensor * ggml_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_type type); + + // make contiguous + GGML_API struct ggml_tensor * ggml_cont( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // make contiguous, with new shape + GGML_API struct ggml_tensor * ggml_cont_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_cont_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_cont_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_cont_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // return view(a), b specifies the new shape + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_reshape_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_reshape_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // offset in bytes + GGML_API struct ggml_tensor * ggml_view_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + size_t nb1, // row stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_permute( + struct ggml_context * ctx, + struct ggml_tensor * a, + int axis0, + int axis1, + int axis2, + int axis3); + + // alias for ggml_permute(ctx, a, 1, 0, 2, 3) + GGML_API struct ggml_tensor * ggml_transpose( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // supports 4D a: + // a [n_embd, ne1, ne2, ne3] + // b I32 [n_rows, ne2, ne3, 1] + // + // return [n_embd, n_rows, ne2, ne3] + GGML_API struct ggml_tensor * ggml_get_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // data + struct ggml_tensor * b); // row indices + + GGML_API struct ggml_tensor * ggml_get_rows_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_get_rows result + struct ggml_tensor * b, // row indices + struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape + + // a TD [n_embd, ne1, ne2, ne3] + // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3 + // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1) + // + // undefined behavior if destination rows overlap + // + // broadcast: + // ne2 % ne11 == 0 + // ne3 % ne12 == 0 + // + // return view(a) + GGML_API struct ggml_tensor * ggml_set_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // destination + struct ggml_tensor * b, // source + struct ggml_tensor * c); // row indices + + GGML_API struct ggml_tensor * ggml_diag( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // set elements above the diagonal to -INF + GGML_API struct ggml_tensor * ggml_diag_mask_inf( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // set elements above the diagonal to 0 + GGML_API struct ggml_tensor * ggml_diag_mask_zero( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + GGML_API struct ggml_tensor * ggml_soft_max( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a [ne0, ne01, ne02, ne03] + // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional + // + // broadcast: + // ne02 % ne12 == 0 + // ne03 % ne13 == 0 + // + // fused soft_max(a*scale + mask*(ALiBi slope)) + // max_bias = 0.0f for no ALiBi + GGML_API struct ggml_tensor * ggml_soft_max_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API void ggml_soft_max_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // rotary position embedding + // if (mode & 1) - skip n_past elements (NOT SUPPORTED) + // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style + // + // b is an int32 vector with size a->ne[2], it contains the positions + GGML_API struct ggml_tensor * ggml_rope( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // RoPE operations with extended options + // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token] + // b is an int32 vector with size n_token + // c is freq factors (e.g. phi3-128k), (optional) + // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi + // + // pseudo-code for computing theta: + // for i in [0, n_dims/2): + // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims); + // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c + // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here + // + // other params are used by YaRN RoPE scaling, these default values will disable YaRN: + // freq_scale = 1.0f + // ext_factor = 0.0f + // attn_factor = 1.0f + // beta_fast = 0.0f + // beta_slow = 0.0f + // + // example: + // (marking: c = cos, s = sin, 0 = unrotated) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000] + // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs] + // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000] + // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss] + GGML_API struct ggml_tensor * ggml_rope_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // multi-dimensional RoPE, for Qwen-VL and similar vision models + // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX + // sections specify how many dimensions to rotate in each section: + // section length is equivalent to number of cos/sin pairs, NOT the number of dims + // (i.e. sum of 4 sections are expected to be n_dims/2) + // last sections can be 0, means ignored + // all other options are identical to ggml_rope_ext + // + // important note: + // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION + // if you need normal ordering, there are 2 methods: + // (1) split the tensor manually using ggml_view + // (2) permute the weight upon conversion + // - for VISION, n_dims must be head_size/2 + // + // example M-RoPE: + // given sections = [t=4, y=2, x=2, 0] + // given a single head with size = 18 --> [000000000000000000] + // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering) + // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering) + // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section + // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section + // + // example vision RoPE: + // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx] + // other values of n_dims are untested and is undefined behavior + // note: unlike MROPE, the theta for each dim is computed differently for each section + // in other words, idx used for theta: [0123] for y section, then [0123] for x section + GGML_API struct ggml_tensor * ggml_rope_multi( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext_inplace instead"); + + // compute correction dims for YaRN RoPE scaling + GGML_API void ggml_rope_yarn_corr_dims( + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]); + + // rotary position embedding backward, i.e compute dx from dy + // a - dy + GGML_API struct ggml_tensor * ggml_rope_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_rope result + struct ggml_tensor * b, // positions + struct ggml_tensor * c, // freq factors + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[4], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + + // clamp + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_clamp( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + + // im2col + // converts data into a format that effectively results in a convolution when combined with matrix multiplication + GGML_API struct ggml_tensor * ggml_im2col( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D, + enum ggml_type dst_type); + + GGML_API struct ggml_tensor * ggml_im2col_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // gradient of im2col output + int64_t * ne, // shape of im2col input + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D); + + // col2im_1d: scatter-add GEMM columns back to 1D signal + // a: [K*OC, T_in] (columns from matmul, K = a->ne[0]/OC) + // result: [T_out, OC] where T_out = (T_in - 1)*s0 + K - 2*p0 + GGML_API struct ggml_tensor * ggml_col2im_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // columns [K*OC, T_in] + int s0, // stride + int oc, // output channels + int p0); // padding to crop from both sides + + GGML_API struct ggml_tensor * ggml_conv_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + // conv_1d with padding = half + // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d) + GGML_API struct ggml_tensor* ggml_conv_1d_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s, // stride + int d); // dilation + + // depthwise + // TODO: this is very likely wrong for some cases! - needs more testing + GGML_API struct ggml_tensor * ggml_conv_1d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_transpose_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_im2col_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2, // dilation depth + enum ggml_type dst_type); + + // a: [OC*IC, KD, KH, KW] + // b: [N*IC, ID, IH, IW] + // result: [N*OC, OD, OH, OW] + GGML_API struct ggml_tensor * ggml_conv_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2 // dilation depth + ); + + // kernel size is a->ne[0] x a->ne[1] + // stride is equal to kernel size + // padding is zero + // example: + // a: 16 16 3 768 + // b: 1024 1024 3 1 + // res: 64 64 768 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // kernel size is a->ne[0] x a->ne[1] + // stride is 1 + // padding is half + // example: + // a: 3 3 256 256 + // b: 64 64 256 1 + // res: 64 64 256 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // depthwise (via im2col and mul_mat) + GGML_API struct ggml_tensor * ggml_conv_2d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + // Depthwise 2D convolution + // may be faster than ggml_conv_2d_dw, but not available in all backends + // a: KW KH 1 C convolution kernel + // b: W H C N input data + // res: W_out H_out C N + GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride0, + int stride1, + int pad0, + int pad1, + int dilation0, + int dilation1); + + GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride); + + GGML_API struct ggml_tensor * ggml_conv_2d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC] + struct ggml_tensor * b, // input data [W, H, C, N] + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_conv_3d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC] + struct ggml_tensor * b, // input [W, H, D, C * N] + int s0, // stride + int s1, + int s2, + int p0, // padding + int p1, + int p2, + int d0, // dilation + int d1, + int d2, + int n_channels, + int n_batch, + int n_channels_out); + + enum ggml_op_pool { + GGML_OP_POOL_MAX, + GGML_OP_POOL_AVG, + GGML_OP_POOL_COUNT, + }; + + GGML_API struct ggml_tensor * ggml_pool_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, // kernel size + int s0, // stride + int p0); // padding + + // the result will have 2*p0 padding for the first dimension + // and 2*p1 padding for the second dimension + GGML_API struct ggml_tensor * ggml_pool_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + GGML_API struct ggml_tensor * ggml_pool_2d_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * af, // "a"/input used in forward pass + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + enum ggml_scale_mode { + GGML_SCALE_MODE_NEAREST = 0, + GGML_SCALE_MODE_BILINEAR = 1, + GGML_SCALE_MODE_BICUBIC = 2, + + GGML_SCALE_MODE_COUNT + }; + + enum ggml_scale_flag { + GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8), + GGML_SCALE_FLAG_ANTIALIAS = (1 << 9), + }; + + // interpolate + // multiplies ne0 and ne1 by scale factor + GGML_API struct ggml_tensor * ggml_upscale( + struct ggml_context * ctx, + struct ggml_tensor * a, + int scale_factor, + enum ggml_scale_mode mode); + + // interpolate + // interpolate scale to specified dimensions + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int ne0, + int ne1, + int ne2, + int ne3, + enum ggml_scale_mode mode), + "use ggml_interpolate instead"); + + // Up- or downsamples the input to the specified size. + // 2D scale modes (eg. bilinear) are applied to the first two dimensions. + GGML_API struct ggml_tensor * ggml_interpolate( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...] + + // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0] + GGML_API struct ggml_tensor * ggml_pad( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + GGML_API struct ggml_tensor * ggml_pad_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3 + ); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_ext_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3); + + // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c] + GGML_API struct ggml_tensor * ggml_pad_reflect_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1); + + // Move tensor elements by an offset given for each dimension. Elements that + // are shifted beyond the last position are wrapped around to the beginning. + GGML_API struct ggml_tensor * ggml_roll( + struct ggml_context * ctx, + struct ggml_tensor * a, + int shift0, + int shift1, + int shift2, + int shift3); + + // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing + // zeroes everywhere outside the masked area + GGML_API struct ggml_tensor * ggml_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_tri_type type); + + // Fill tensor a with constant c + GGML_API struct ggml_tensor * ggml_fill( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + GGML_API struct ggml_tensor * ggml_fill_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151 + // timesteps: [N,] + // return: [N, dim] + GGML_API struct ggml_tensor * ggml_timestep_embedding( + struct ggml_context * ctx, + struct ggml_tensor * timesteps, + int dim, + int max_period); + + // sort rows + enum ggml_sort_order { + GGML_SORT_ORDER_ASC, + GGML_SORT_ORDER_DESC, + }; + + GGML_API struct ggml_tensor * ggml_argsort( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_sort_order order); + + // similar to ggml_top_k but implemented as `argsort` + `view` + GGML_API struct ggml_tensor * ggml_argsort_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + // top k elements per row + // note: the resulting top k indices are in no particular order + GGML_API struct ggml_tensor * ggml_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + GGML_API struct ggml_tensor * ggml_arange( + struct ggml_context * ctx, + float start, + float stop, + float step); + + // q: [n_embd_k, n_batch, n_head, ne3 ] + // k: [n_embd_k, n_kv, n_head_kv, ne3 ] + // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !! + // mask: [n_kv, n_batch, ne32, ne33] + // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !! + // + // broadcast: + // n_head % n_head_kv == 0 + // n_head % ne32 == 0 + // ne3 % ne33 == 0 + // + GGML_API struct ggml_tensor * ggml_flash_attn_ext( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + float scale, + float max_bias, + float logit_softcap); + + GGML_API void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( + const struct ggml_tensor * a); + + GGML_API void ggml_flash_attn_ext_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + // TODO: needs to be adapted to ggml_flash_attn_ext + GGML_API struct ggml_tensor * ggml_flash_attn_back( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * d, + bool masked); + + GGML_API struct ggml_tensor * ggml_ssm_conv( + struct ggml_context * ctx, + struct ggml_tensor * sx, + struct ggml_tensor * c); + + GGML_API struct ggml_tensor * ggml_ssm_scan( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * dt, + struct ggml_tensor * A, + struct ggml_tensor * B, + struct ggml_tensor * C, + struct ggml_tensor * ids); + + // partition into non-overlapping windows with padding if needed + // example: + // a: 768 64 64 1 + // w: 14 + // res: 768 14 14 25 + // used in sam + GGML_API struct ggml_tensor * ggml_win_part( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w); + + // reverse of ggml_win_part + // used in sam + GGML_API struct ggml_tensor * ggml_win_unpart( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w0, + int h0, + int w); + + GGML_API struct ggml_tensor * ggml_unary( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + GGML_API struct ggml_tensor * ggml_unary_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + // used in sam + GGML_API struct ggml_tensor * ggml_get_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + int qh, + int kh); + + // used in sam + GGML_API struct ggml_tensor * ggml_add_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv6( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * r, + struct ggml_tensor * tf, + struct ggml_tensor * td, + struct ggml_tensor * state); + + GGML_API struct ggml_tensor * ggml_gated_linear_attn( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * q, + struct ggml_tensor * g, + struct ggml_tensor * state, + float scale); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state); + + /* Solves a specific equation of the form Ax=B, where A is a triangular matrix + * without zeroes on the diagonal (i.e. invertible). + * B can have any number of columns, but must have the same number of rows as A + * If A is [n, n] and B is [n, m], then the result will be [n, m] as well + * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases + * where n > 100 sparingly, pre-chunk if necessary. + * + * If left = false, solves xA=B instead + * If lower = false, assumes upper triangular instead + * If uni = true, assumes diagonal of A to be all ones (will override actual values) + * + * TODO: currently only lower, right, non-unitriangular variant is implemented + */ + GGML_API struct ggml_tensor * ggml_solve_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + bool left, + bool lower, + bool uni); + + // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST] + // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306 + // + // state is a 3D tensor of shape (S_v*S_v*H, K, n_seqs): + // K == 1: output carries the final state only. + // K > 1: output carries K snapshot slots; the kernel writes the last min(n_tokens, K) + // per-token snapshots into the trailing slots + GGML_API struct ggml_tensor * ggml_gated_delta_net( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * g, + struct ggml_tensor * beta, + struct ggml_tensor * state); + + // custom operators + + typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata); + typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata); + typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata); + +#define GGML_N_TASKS_MAX (-1) + // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks + + GGML_API struct ggml_tensor * ggml_map_custom1( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + // loss function + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b); // labels + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b, // labels + struct ggml_tensor * c); // gradients of cross_entropy_loss result + + // AdamW optimizer step + // Paper: https://arxiv.org/pdf/1711.05101v3.pdf + // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html + GGML_API struct ggml_tensor * ggml_opt_step_adamw( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * m, + struct ggml_tensor * v, + struct ggml_tensor * adamw_params); // parameters such as the learning rate + + // stochastic gradient descent step (with weight decay) + GGML_API struct ggml_tensor * ggml_opt_step_sgd( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * sgd_params); // alpha, weight decay + + // build forward multiple tensors and select one of them for computing + // this is useful for creating graphs that have constant topology but compute different things based on the input + // ref: https://github.com/ggml-org/llama.cpp/pull/18550 + // + // nodes: + // | - build forward into the graph but do not compute + // c - build forward into the graph and compute + // + // | | ... c ... | + // | | ... c ... | + // | | ... c ... | + // [0 1 ... idx ... n-1] <-- ggml_build_forward_select(..., n, idx) + // c + // c + // + // example: + // struct ggml_tensor * curs[3]; + // + // curs[0] = compute0(...); + // curs[1] = compute1(...); + // curs[2] = compute2(...); + // + // int idx = select_branch(some_input); + // + // struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx); + // + GGML_API struct ggml_tensor * ggml_build_forward_select( + struct ggml_cgraph * cgraph, + struct ggml_tensor ** tensors, + int n_tensors, + int idx); + + GGML_API void ggml_build_forward_expand( + struct ggml_cgraph * cgraph, + struct ggml_tensor * tensor); + + GGML_API void ggml_build_backward_expand( + struct ggml_context * ctx, // context for gradient computation + struct ggml_cgraph * cgraph, + struct ggml_tensor ** grad_accs); + + // graph allocation in a context + GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false + GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads); + GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads); + GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst); + GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1 + GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph); + + GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph); + GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i] + GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph); + GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph); + + GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor); + + GGML_API size_t ggml_graph_overhead(void); + GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads); + + GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name); + GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + + // print info and performance information for the graph + GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph); + + // dump the graph into a file using the dot format + GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename); + + // TODO these functions were sandwiched in the old optimization interface, is there a better place for them? + typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data); + GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data); + + GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor); + + // + // quantization + // + + // - ggml_quantize_init can be called multiple times with the same type + // it will only initialize the quantization tables for the first call or after ggml_quantize_free + // automatically called by ggml_quantize_chunk for convenience + // + // - ggml_quantize_free will free any memory allocated by ggml_quantize_init + // call this at the end of the program to avoid memory leaks + // + // note: these are thread-safe + // + GGML_API void ggml_quantize_init(enum ggml_type type); + GGML_API void ggml_quantize_free(void); + + // some quantization type cannot be used without an importance matrix + GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type); + + // calls ggml_quantize_init internally (i.e. can allocate memory) + GGML_API size_t ggml_quantize_chunk( + enum ggml_type type, + const float * src, + void * dst, + int64_t start, + int64_t nrows, + int64_t n_per_row, + const float * imatrix); + +#ifdef __cplusplus + // restrict not standard in C++ +# if defined(__GNUC__) +# define GGML_RESTRICT __restrict__ +# elif defined(__clang__) +# define GGML_RESTRICT __restrict +# elif defined(_MSC_VER) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT +# endif +#else +# if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT restrict +# endif +#endif + typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + + struct ggml_type_traits { + const char * type_name; + int64_t blck_size; + int64_t blck_size_interleave; // interleave elements in blocks + size_t type_size; + bool is_quantized; + ggml_to_float_t to_float; + ggml_from_float_t from_float_ref; + }; + + GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type); + + // ggml threadpool + // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend + // the goal should be to create an API that other backends can use move everything to the ggml base + + // scheduling priorities + enum ggml_sched_priority { + GGML_SCHED_PRIO_LOW = -1, + GGML_SCHED_PRIO_NORMAL, + GGML_SCHED_PRIO_MEDIUM, + GGML_SCHED_PRIO_HIGH, + GGML_SCHED_PRIO_REALTIME + }; + + // threadpool params + // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults + struct ggml_threadpool_params { + bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings) + int n_threads; // number of threads + enum ggml_sched_priority prio; // thread priority + uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling) + bool strict_cpu; // strict cpu placement + bool paused; // start in paused state + }; + + struct ggml_threadpool; // forward declaration, see ggml.c + + typedef struct ggml_threadpool * ggml_threadpool_t; + + GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads); + GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads); + GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/gguf.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/gguf.h new file mode 100644 index 00000000..67851ba6 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/gguf.h @@ -0,0 +1,210 @@ +// This file contains functionality related to "GGUF" files, the binary file format used by ggml. +// GGUF files have the following structure: +// +// 1. File magic "GGUF" (4 bytes). +// 2. File version (uint32_t). +// 3. Number of ggml tensors in file (int64_t). +// 4. Number of key-value-pairs in file (int64_t). +// 5. For each KV pair: +// 1. The key (string). +// 2. The value type (gguf_type). +// 3a. If the value type is GGUF_TYPE_ARRAY: +// 1. The type of the array (gguf_type). +// 2. The number of elements in the array (uint64_t). +// 3. The binary representation of each element in the array. +// 3b. Otherwise: +// 1. The binary representation of the value. +// 6. For each ggml tensor: +// 1. The tensor name (string). +// 2. The number of dimensions of the tensor (uint32_t). +// 3. For each dimension: +// 1. The size of the tensor in the dimension (int64_t). +// 4. The tensor data type (ggml_type). +// 5. The tensor data offset in the tensor data binary blob (uint64_t). +// 7. The tensor data binary blob (optional, aligned). +// +// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator. +// All enums are stored as int32_t. +// All bool values are stored as int8_t. +// If the special key "general.alignment" (uint32_t) is defined it is used for alignment, +// otherwise GGUF_DEFAULT_ALIGNMENT is used. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" + +#include +#include + +#define GGUF_MAGIC "GGUF" +#define GGUF_VERSION 3 + +#define GGUF_KEY_GENERAL_ALIGNMENT "general.alignment" + +#define GGUF_DEFAULT_ALIGNMENT 32 + +#ifdef __cplusplus +extern "C" { +#endif + + // types that can be stored as GGUF KV data + enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, + GGUF_TYPE_UINT64 = 10, + GGUF_TYPE_INT64 = 11, + GGUF_TYPE_FLOAT64 = 12, + GGUF_TYPE_COUNT, // marks the end of the enum + }; + + struct gguf_context; + + struct gguf_init_params { + bool no_alloc; + + // if not NULL, create a ggml_context and allocate the tensor data in it + struct ggml_context ** ctx; + }; + + // callback to simulate or wrap a FILE pointer - read up to `len` bytes at `offset` into `output` and return the number of bytes read + typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len); + + GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params); + + // max_chunk_read is the maximum number of bytes that the GGUF code will read at once from the callback, a value of 0 means no limit + GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params); + + GGML_API void gguf_free(struct gguf_context * ctx); + + GGML_API const char * gguf_type_name(enum gguf_type type); + + GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx); + GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx); + GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); // padded to gguf_get_alignment if and only if the gguf_context contains at least one tensor + + GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx); + GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found + GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id); + + GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id); + GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id); + + // will abort if the wrong type is used for the key + GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id); + GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id); + GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id); + GGML_API size_t gguf_get_arr_n (const struct gguf_context * ctx, int64_t key_id); + + // get raw pointer to the first element of the array with the given key_id + // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference) + GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id); + + // get ith C string from array with given key_id + GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i); + + GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx); + GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found + GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id); + GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id); + + // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist) + GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key); + + // overrides an existing KV pair or adds a new one, the new KV pair is always at the back + GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val); + GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val); + GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val); + GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val); + GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val); + GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val); + GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val); + GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val); + GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val); + GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val); + GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val); + GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val); + + // creates a new array with n elements of the given type and copies the corresponding number of bytes from data + GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n); + + // creates a new array with n strings and copies the corresponding strings from data + GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n); + + // set or add KV pairs from another context + GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src); + + // add tensor to GGUF context, tensor name must be unique + GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor); + + // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated + // in such a way that the tensor data remains as one contiguous block (except for padding) + GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type); + + // assumes that at least gguf_get_tensor_size bytes can be read from data + GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data); + + // writing gguf files can be done in 3 ways: + // + // - write the entire gguf_context to a binary file in a single pass: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ false); + // + // - write only the meta data to a file, then re-open the file and append the tensor data: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ true); + // FILE * f = fopen(fname, "ab"); + // fwrite(f, ...); // write tensor data + // fclose(f); + // + // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data: + // + // FILE * f = fopen(fname, "wb"); + // const size_t size_meta = gguf_get_meta_size(ctx); + // fseek(f, size_meta, SEEK_SET); + // fwrite(f, ...); // write tensor data + // void * data = malloc(size_meta); + // gguf_get_meta_data(ctx, data); + // rewind(f); + // fwrite(data, 1, data, f); + // free(data); + // fclose(f); + // + + // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); + GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); + + // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding + GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx); + + // writes the meta data to pointer "data" + GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data); + +#ifdef __cplusplus +} +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/llama.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/llama.h new file mode 100644 index 00000000..27e48067 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/llama.h @@ -0,0 +1,1588 @@ +#ifndef LLAMA_H +#define LLAMA_H + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-backend.h" +#include "ggml-opt.h" +#include "gguf.h" + +#include +#include +#include +#include + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define LLAMA_API __declspec(dllexport) +# else +# define LLAMA_API __declspec(dllimport) +# endif +# else +# define LLAMA_API __attribute__ ((visibility ("default"))) +# endif +#else +# define LLAMA_API +#endif + +#ifdef __GNUC__ +# define DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define DEPRECATED(func, hint) func +#endif + +#define LLAMA_DEFAULT_SEED 0xFFFFFFFF + +#define LLAMA_TOKEN_NULL -1 + +#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' +#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' +#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq' + +#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN +#define LLAMA_SESSION_VERSION 9 + +#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ +#define LLAMA_STATE_SEQ_VERSION 2 + +#ifdef __cplusplus +extern "C" { +#endif + + // + // C interface + // + // TODO: show sample usage + // + + struct llama_vocab; + struct llama_model; + struct llama_context; + struct llama_sampler; + + typedef struct llama_memory_i * llama_memory_t; + + typedef int32_t llama_pos; + typedef int32_t llama_token; + typedef int32_t llama_seq_id; + + enum llama_vocab_type { + LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab + LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback + LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE + LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece + LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram + LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization + LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming + }; + + enum llama_rope_type { + LLAMA_ROPE_TYPE_NONE = -1, + LLAMA_ROPE_TYPE_NORM = 0, + LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, + LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE, + LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE, + LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION, + }; + + enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file + LLAMA_TOKEN_TYPE_UNDEFINED = 0, + LLAMA_TOKEN_TYPE_NORMAL = 1, + LLAMA_TOKEN_TYPE_UNKNOWN = 2, + LLAMA_TOKEN_TYPE_CONTROL = 3, + LLAMA_TOKEN_TYPE_USER_DEFINED = 4, + LLAMA_TOKEN_TYPE_UNUSED = 5, + LLAMA_TOKEN_TYPE_BYTE = 6, + }; + + enum llama_token_attr { + LLAMA_TOKEN_ATTR_UNDEFINED = 0, + LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0, + LLAMA_TOKEN_ATTR_UNUSED = 1 << 1, + LLAMA_TOKEN_ATTR_NORMAL = 1 << 2, + LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL? + LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4, + LLAMA_TOKEN_ATTR_BYTE = 1 << 5, + LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6, + LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7, + LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8, + LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9, + }; + + // model file types + enum llama_ftype { + LLAMA_FTYPE_ALL_F32 = 0, + LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed + // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed + LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors + LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors + //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack + //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack + //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack + LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors + LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors + LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors + LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors + + LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file + }; + + enum llama_rope_scaling_type { + LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, + LLAMA_ROPE_SCALING_TYPE_NONE = 0, + LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, + LLAMA_ROPE_SCALING_TYPE_YARN = 2, + LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, + LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE, + }; + + enum llama_pooling_type { + LLAMA_POOLING_TYPE_UNSPECIFIED = -1, + LLAMA_POOLING_TYPE_NONE = 0, + LLAMA_POOLING_TYPE_MEAN = 1, + LLAMA_POOLING_TYPE_CLS = 2, + LLAMA_POOLING_TYPE_LAST = 3, + LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph + }; + + enum llama_attention_type { + LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1, + LLAMA_ATTENTION_TYPE_CAUSAL = 0, + LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1, + }; + + enum llama_flash_attn_type { + LLAMA_FLASH_ATTN_TYPE_AUTO = -1, + LLAMA_FLASH_ATTN_TYPE_DISABLED = 0, + LLAMA_FLASH_ATTN_TYPE_ENABLED = 1, + }; + + LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); + + enum llama_split_mode { + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_TENSOR = 3, + }; + + enum llama_context_type { + LLAMA_CONTEXT_TYPE_DEFAULT = 0, + LLAMA_CONTEXT_TYPE_MTP = 1, + }; + + // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) + typedef struct llama_token_data { + llama_token id; // token id + float logit; // log-odds of the token + float p; // probability of the token + } llama_token_data; + + typedef struct llama_token_data_array { + // TODO: consider SoA + // NOTE: this pointer can be modified by the samplers + llama_token_data * data; + size_t size; + int64_t selected; // this is the index in the data array (i.e. not the token id) + bool sorted; // note: do not assume the data is sorted - always check this flag + } llama_token_data_array; + + typedef bool (*llama_progress_callback)(float progress, void * user_data); + + // Input data for llama_encode/llama_decode + // A llama_batch object can contain input about one or many sequences + // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens + // + // - token : the token ids of the input (used when embd is NULL) + // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) + // - pos : the positions of the respective token in the sequence + // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode) + // - seq_id : the sequence to which the respective token belongs + // (if set to NULL, the sequence ID will be assumed to be 0) + // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output + // (if set to NULL: + // - if embeddings: all tokens are output + // - if not: only the last token is output + // ) + // + typedef struct llama_batch { + int32_t n_tokens; + + llama_token * token; + float * embd; + llama_pos * pos; + int32_t * n_seq_id; + llama_seq_id ** seq_id; + int8_t * logits; // TODO: rename this to "output" + } llama_batch; + + enum llama_model_kv_override_type { + LLAMA_KV_OVERRIDE_TYPE_INT, + LLAMA_KV_OVERRIDE_TYPE_FLOAT, + LLAMA_KV_OVERRIDE_TYPE_BOOL, + LLAMA_KV_OVERRIDE_TYPE_STR, + }; + + enum llama_model_meta_key { + LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE, + LLAMA_MODEL_META_KEY_SAMPLING_TOP_K, + LLAMA_MODEL_META_KEY_SAMPLING_TOP_P, + LLAMA_MODEL_META_KEY_SAMPLING_MIN_P, + LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY, + LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD, + LLAMA_MODEL_META_KEY_SAMPLING_TEMP, + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N, + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA, + }; + + struct llama_model_kv_override { + enum llama_model_kv_override_type tag; + + char key[128]; + + union { + int64_t val_i64; + double val_f64; + bool val_bool; + char val_str[128]; + }; + }; + + struct llama_model_tensor_buft_override { + const char * pattern; + ggml_backend_buffer_type_t buft; + }; + + struct llama_model_params { + // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) + ggml_backend_dev_t * devices; + + // NULL-terminated list of buffer types to use for tensors that match a pattern + const struct llama_model_tensor_buft_override * tensor_buft_overrides; + + int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers + enum llama_split_mode split_mode; // how to split the model across multiple GPUs + + // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE + int32_t main_gpu; + + // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() + const float * tensor_split; + + // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. + // If the provided progress_callback returns true, model loading continues. + // If it returns false, model loading is immediately aborted. + llama_progress_callback progress_callback; + + // context pointer passed to the progress callback + void * progress_callback_user_data; + + // override key-value pairs of the model meta data + const struct llama_model_kv_override * kv_overrides; + + // Keep the booleans together to avoid misalignment during copy-by-value. + bool vocab_only; // only load the vocabulary, no weights + bool use_mmap; // use mmap if possible + bool use_direct_io; // use direct io, takes precedence over use_mmap when supported + bool use_mlock; // force system to keep model in RAM + bool check_tensors; // validate model tensor data + bool use_extra_bufts; // use extra buffer types (used for weight repacking) + bool no_host; // bypass host buffer allowing extra buffers to be used + bool no_alloc; // only load metadata and simulate memory allocations + }; + + struct llama_sampler_seq_config { + llama_seq_id seq_id; + struct llama_sampler * sampler; + }; + + // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations + // https://github.com/ggml-org/llama.cpp/pull/7544 + struct llama_context_params { + uint32_t n_ctx; // text context, 0 = from model + uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode + uint32_t n_ubatch; // physical maximum batch size + uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) + uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing + + enum llama_context_type ctx_type; // set the context type (e.g. MTP) + enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` + enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id + enum llama_attention_type attention_type; // attention type to use for embeddings + enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention + + // ref: https://github.com/ggml-org/llama.cpp/pull/2054 + float rope_freq_base; // RoPE base frequency, 0 = from model + float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model + float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model + float yarn_attn_factor; // YaRN magnitude scaling factor + float yarn_beta_fast; // YaRN low correction dim + float yarn_beta_slow; // YaRN high correction dim + uint32_t yarn_orig_ctx; // YaRN original context size + float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default) + + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; + + enum ggml_type type_k; // data type for K cache [EXPERIMENTAL] + enum ggml_type type_v; // data type for V cache [EXPERIMENTAL] + + // Abort callback + // if it returns true, execution of llama_decode() will be aborted + // currently works only with CPU execution + ggml_abort_callback abort_callback; + void * abort_callback_data; + + // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. + bool embeddings; // if true, extract embeddings (together with logits) + bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU + bool no_perf; // measure performance timings + bool op_offload; // offload host tensor operations to device + bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) + // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases + // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 + bool kv_unified; // use a unified buffer across the input sequences when computing the attention + // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix + // ref: https://github.com/ggml-org/llama.cpp/pull/14363 + + // [EXPERIMENTAL] + // backend sampler chain configuration (make sure the caller keeps the sampler chains alive) + // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init) + struct llama_sampler_seq_config * samplers; + size_t n_samplers; + + // a source/target/parent context + // can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts + struct llama_context * ctx_other; + }; + + struct llama_model_tensor_override { + const char * pattern; + enum ggml_type type; + }; + + struct llama_model_imatrix_data { + const char * name; + const float * data; + size_t size; + }; + + // model quantization parameters + typedef struct llama_model_quantize_params { + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // token embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + bool keep_split; // quantize to the same number of shards + bool dry_run; // calculate and show the final quantization size without performing quantization + const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data + const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides + const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides + const int32_t * prune_layers; // pointer to layer indices to prune + } llama_model_quantize_params; + + typedef struct llama_logit_bias { + llama_token token; + float bias; + } llama_logit_bias; + + typedef struct llama_sampler_chain_params { + bool no_perf; // whether to measure performance timings + } llama_sampler_chain_params; + + // used in chat template + typedef struct llama_chat_message { + const char * role; + const char * content; + } llama_chat_message; + + // lora adapter + struct llama_adapter_lora; + + // Helpers for getting default parameters + // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172) + LLAMA_API struct llama_model_params llama_model_default_params(void); + LLAMA_API struct llama_context_params llama_context_default_params(void); + LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void); + LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void); + + // Initialize the llama + ggml backend + // If numa is true, use NUMA optimizations + // Call once at the start of the program + LLAMA_API void llama_backend_init(void); + + // Call once at the end of the program - currently only used for MPI + LLAMA_API void llama_backend_free(void); + + //optional: + LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa); + + // Optional: an auto threadpool gets created in ggml if not passed explicitly + LLAMA_API void llama_attach_threadpool( + struct llama_context * ctx, + ggml_threadpool_t threadpool, + ggml_threadpool_t threadpool_batch); + + LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); + + typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata); + + // Create a new model from GGUF metadata as well as a function to set the tensor data + // - tensors are created as GGML_TYPE_F32 by default, + // override by adding a tensor with the same name but a different name to the context + LLAMA_API struct llama_model * llama_model_init_from_user( + struct gguf_context * metadata, + llama_model_set_tensor_data_t set_tensor_data, // function to initialize tensor data with + void * set_tensor_data_ud, // userdata for function + struct llama_model_params params); + + DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( + const char * path_model, + struct llama_model_params params), + "use llama_model_load_from_file instead"); + + // Load a model from a file + // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf + // If the split file name does not follow this pattern, use llama_model_load_from_splits + LLAMA_API struct llama_model * llama_model_load_from_file( + const char * path_model, + struct llama_model_params params); + + // Load a model from an open FILE pointer + LLAMA_API struct llama_model * llama_model_load_from_file_ptr( + FILE * file, + struct llama_model_params params); + + // Load a model from multiple splits (support custom naming scheme) + // The paths must be in the correct order + LLAMA_API struct llama_model * llama_model_load_from_splits( + const char ** paths, + size_t n_paths, + struct llama_model_params params); + + LLAMA_API void llama_model_save_to_file( + const struct llama_model * model, + const char * path_model); + + DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model), + "use llama_model_free instead"); + + LLAMA_API void llama_model_free(struct llama_model * model); + + LLAMA_API struct llama_context * llama_init_from_model( + struct llama_model * model, + struct llama_context_params params); + + DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model( + struct llama_model * model, + struct llama_context_params params), + "use llama_init_from_model instead"); + + // Frees all allocated memory + LLAMA_API void llama_free(struct llama_context * ctx); + + LLAMA_API int64_t llama_time_us(void); + + LLAMA_API size_t llama_max_devices(void); + LLAMA_API size_t llama_max_parallel_sequences(void); + LLAMA_API size_t llama_max_tensor_buft_overrides(void); + + LLAMA_API bool llama_supports_mmap (void); + LLAMA_API bool llama_supports_mlock (void); + LLAMA_API bool llama_supports_gpu_offload(void); + LLAMA_API bool llama_supports_rpc (void); + + // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions + // In some cases the requested values via llama_context_params may differ from the actual values used by the context + // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732 + LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx); + + DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); + DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); + DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); + DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); + + DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); + + LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); + LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx); + LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type + + LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); + LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model); + + LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model); + + // Get the model's RoPE frequency scaling factor + LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); + + // Returns the number of classifier outputs (only valid for classifier models) + // Undefined behavior for non-classifier models + LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); + + // Returns label of classifier output by index ( 1` + // p0 < 0 : [0, p1] + // p1 < 0 : [p0, inf) + LLAMA_API void llama_memory_seq_div( + llama_memory_t mem, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d); + + // Returns the smallest position present in the memory for the specified sequence + // This is typically non-zero only for SWA caches + // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory + // Return -1 if the sequence is empty + LLAMA_API llama_pos llama_memory_seq_pos_min( + llama_memory_t mem, + llama_seq_id seq_id); + + // Returns the largest position present in the memory for the specified sequence + // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory + // Return -1 if the sequence is empty + LLAMA_API llama_pos llama_memory_seq_pos_max( + llama_memory_t mem, + llama_seq_id seq_id); + + // Check if the memory supports shifting + LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); + + // + // State / sessions + // + + // Returns the *actual* size in bytes of the state + // (logits, embedding and memory) + // Only use when saving the state, not when restoring it, otherwise the size may be too small. + LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); + LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx), + "use llama_state_get_size instead"); + + // Copies the state to the specified destination address. + // Destination needs to have allocated enough memory. + // Returns the number of bytes copied + LLAMA_API size_t llama_state_get_data( + struct llama_context * ctx, + uint8_t * dst, + size_t size); + LLAMA_API DEPRECATED(size_t llama_copy_state_data( + struct llama_context * ctx, + uint8_t * dst), + "use llama_state_get_data instead"); + + // Set the state reading from the specified address + // Returns the number of bytes read + LLAMA_API size_t llama_state_set_data( + struct llama_context * ctx, + const uint8_t * src, + size_t size); + LLAMA_API DEPRECATED(size_t llama_set_state_data( + struct llama_context * ctx, + const uint8_t * src), + "use llama_state_set_data instead"); + + // Save/load session file + LLAMA_API bool llama_state_load_file( + struct llama_context * ctx, + const char * path_session, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + LLAMA_API DEPRECATED(bool llama_load_session_file( + struct llama_context * ctx, + const char * path_session, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out), + "use llama_state_load_file instead"); + + LLAMA_API bool llama_state_save_file( + struct llama_context * ctx, + const char * path_session, + const llama_token * tokens, + size_t n_token_count); + LLAMA_API DEPRECATED(bool llama_save_session_file( + struct llama_context * ctx, + const char * path_session, + const llama_token * tokens, + size_t n_token_count), + "use llama_state_save_file instead"); + + // Get the exact size needed to copy the state of a single sequence + LLAMA_API size_t llama_state_seq_get_size( + struct llama_context * ctx, + llama_seq_id seq_id); + + // Copy the state of a single sequence into the specified buffer + LLAMA_API size_t llama_state_seq_get_data( + struct llama_context * ctx, + uint8_t * dst, + size_t size, + llama_seq_id seq_id); + + // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence + // Returns: + // - Positive: Ok + // - Zero: Failed to load + LLAMA_API size_t llama_state_seq_set_data( + struct llama_context * ctx, + const uint8_t * src, + size_t size, + llama_seq_id dest_seq_id); + + LLAMA_API size_t llama_state_seq_save_file( + struct llama_context * ctx, + const char * filepath, + llama_seq_id seq_id, + const llama_token * tokens, + size_t n_token_count); + + LLAMA_API size_t llama_state_seq_load_file( + struct llama_context * ctx, + const char * filepath, + llama_seq_id dest_seq_id, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + +#define LLAMA_STATE_SEQ_FLAGS_NONE 0 + +// for backwards-compat +#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 + +// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) +#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 + +// Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load). +// Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag. +#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 + + typedef uint32_t llama_state_seq_flags; + + LLAMA_API size_t llama_state_seq_get_size_ext( + struct llama_context * ctx, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_get_data_ext( + struct llama_context * ctx, + uint8_t * dst, + size_t size, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_set_data_ext( + struct llama_context * ctx, + const uint8_t * src, + size_t size, + llama_seq_id dest_seq_id, + llama_state_seq_flags flags); + + // + // Decoding + // + + // Return batch for single sequence of tokens + // The sequence ID will be fixed to 0 + // The position of the tokens will be tracked automatically by llama_decode + // + // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it + // + LLAMA_API struct llama_batch llama_batch_get_one( + llama_token * tokens, + int32_t n_tokens); + + // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens + // Each token can be assigned up to n_seq_max sequence ids + // The batch has to be freed with llama_batch_free() + // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) + // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token + // The rest of the llama_batch members are allocated with size n_tokens + // All members are left uninitialized + LLAMA_API struct llama_batch llama_batch_init( + int32_t n_tokens, + int32_t embd, + int32_t n_seq_max); + + // Frees a batch of tokens allocated with llama_batch_init() + LLAMA_API void llama_batch_free(struct llama_batch batch); + + // Process a batch of tokens. + // In contrast to llama_decode() - this call does not use KV cache. + // For encode-decoder contexts, processes the batch using the encoder. + // Can store the encoder output internally for later use by the decoder's cross-attention layers. + // 0 - success + // < 0 - error. the memory state is restored to the state before this call + LLAMA_API int32_t llama_encode( + struct llama_context * ctx, + struct llama_batch batch); + + // Process a batch of tokens. + // Requires the context to have a memory. + // For encode-decoder contexts, processes the batch using the decoder. + // Positive return values does not mean a fatal error, but rather a warning. + // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context + // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + // Upon other return values, the memory state is restored to the state before this call + // 0 - success + // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) + // 2 - aborted (processed ubatches will remain in the context's memory) + // -1 - invalid input batch + // < -1 - fatal error (processed ubatches will remain in the context's memory) + LLAMA_API int32_t llama_decode( + struct llama_context * ctx, + struct llama_batch batch); + + // Set the number of threads used for decoding + // n_threads is the number of threads used for generation (single token) + // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) + LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch); + + // Get the number of threads used for generation of a single token. + LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); + + // Get the number of threads used for prompt and batch processing (multiple token). + LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); + + // Set whether the context outputs embeddings or not + // TODO: rename to avoid confusion with llama_get_embeddings() + LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); + + // Set whether to use causal attention or not + // If set to true, the model will only attend to the past tokens + LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); + + // Set whether the model is in warmup mode or not + // If true, all model tensors are activated during llama_decode() to load and cache their weights. + // + // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models, + // so it is generally not recommended to use in practice. will be removed in the future + DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup), + "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]"); + + // Set abort callback + LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); + + // Wait until all computations are finished + // This is automatically done when using one of the functions below to obtain the computation results + // and is not necessary to call it explicitly in most cases + LLAMA_API void llama_synchronize(struct llama_context * ctx); + + // Token logits obtained from the last call to llama_decode() + // The logits for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have appeared in the batch. + // Rows: number of tokens for which llama_batch.logits[i] != 0 + // Cols: n_vocab + // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) + LLAMA_API float * llama_get_logits(struct llama_context * ctx); + + // Logits for the ith token. For positive indices, Equivalent to: + // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab + // Negative indices can be used to access logits in reverse order, -1 is the last logit. + // returns NULL for invalid ids. + LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); + + // Get all output token embeddings. + // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, + // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have appeared in the batch. + // shape: [n_outputs*n_embd] + // Otherwise, returns NULL. + // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) + LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); + + // Get the embeddings for the ith token. For positive indices, Equivalent to: + // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd + // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding. + // shape: [n_embd] (1-dimensional) + // returns NULL for invalid ids. + LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); + + // Get the embeddings for a sequence id + // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE + // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence + // otherwise: float[n_embd] (1-dimensional) + LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); + + // + // backend sampling API [EXPERIMENTAL] + // note: use only if the llama_context was created with at least one llama_sampler_seq_config + // + + // Get the backend sampled token for the ith token. + // Returns LLAMA_TOKEN_NULL if no token was sampled. + LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled probabilities for the ith token + // The index matches llama_get_sampled_token_ith(). + // Returns NULL if no probabilities were generated. + LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled logits for the ith token + // Returns NULL if no logits were sampled. + LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled candidates (token ids) for the ith token + // These are needed to map probability/logit indices to vocab token ids. + // Returns NULL if no candidates were sampled. + LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i); + + // + // Vocab + // + + LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token); + + LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token); + + LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token); + + // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) + LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token); + + // Identify if Token Id is a control token or a render-able token + LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token); + + // Special tokens + LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence + LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence + LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn + LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator + LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line + LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding + LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask + + LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab); + LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab); + LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab); + + LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab); + + DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); + DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); + DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); + DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); + DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); + DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); + DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); + DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); + DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); + DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); + DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); + DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); + + // CLS is equivalent to BOS + DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification + "use llama_vocab_bos instead"); + + // + // Tokenization + // + // The API is thread-safe. + // + + /// @details Convert the provided text into tokens. + /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. + /// @return Returns the number of tokens on success, no more than n_tokens_max + /// @return Returns a negative number on failure - the number of tokens that would have been returned + /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit) + /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so. + /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated + /// as plaintext. Does not insert a leading space. + LLAMA_API int32_t llama_tokenize( + const struct llama_vocab * vocab, + const char * text, + int32_t text_len, + llama_token * tokens, + int32_t n_tokens_max, + bool add_special, + bool parse_special); + + // Token Id -> Piece. + // Uses the vocabulary in the provided context. + // Does not write null terminator to the buffer. + // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') + // @param special If true, special tokens are rendered in the output. + LLAMA_API int32_t llama_token_to_piece( + const struct llama_vocab * vocab, + llama_token token, + char * buf, + int32_t length, + int32_t lstrip, + bool special); + + /// @details Convert the provided tokens into text (inverse of llama_tokenize()). + /// @param text The char pointer must be large enough to hold the resulting text. + /// @return Returns the number of chars/bytes on success, no more than text_len_max. + /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned. + /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. + /// @param unparse_special If true, special tokens are rendered in the output. + LLAMA_API int32_t llama_detokenize( + const struct llama_vocab * vocab, + const llama_token * tokens, + int32_t n_tokens, + char * text, + int32_t text_len_max, + bool remove_special, + bool unparse_special); + + // + // Chat templates + // + + /// Apply chat template. Inspired by hf apply_chat_template() on python. + /// + /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template + /// @param tmpl A Jinja template to use for this chat. + /// @param chat Pointer to a list of multiple llama_chat_message + /// @param n_msg Number of llama_chat_message in this chat + /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. + /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages) + /// @param length The size of the allocated buffer + /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. + LLAMA_API int32_t llama_chat_apply_template( + const char * tmpl, + const struct llama_chat_message * chat, + size_t n_msg, + bool add_ass, + char * buf, + int32_t length); + + // Get list of built-in chat templates + LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len); + + // + // Sampling API + // + // Sample usage: + // + // // prepare the sampling chain at the start + // auto sparams = llama_sampler_chain_default_params(); + // + // llama_sampler * smpl = llama_sampler_chain_init(sparams); + // + // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50)); + // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); + // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8)); + // + // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat" + // // this sampler will be responsible to select the actual token + // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); + // + // ... + // + // // decoding loop: + // while (...) { + // ... + // + // llama_decode(ctx, batch); + // + // // sample from the logits of the last token in the batch + // const llama_token id = llama_sampler_sample(smpl, ctx, -1); + // + // ... + // } + // + // llama_sampler_free(smpl); + // + + typedef void * llama_sampler_context_t; + + struct llama_sampler_data { + struct ggml_tensor * logits; + struct ggml_tensor * probs; + struct ggml_tensor * sampled; + struct ggml_tensor * candidates; + }; + + // user code can implement the interface below in order to create custom llama_sampler + struct llama_sampler_i { + const char * (*name) (const struct llama_sampler * smpl); // can be NULL + void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL + void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required + void (*reset) ( struct llama_sampler * smpl); // can be NULL + struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL + void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL + + // [EXPERIMENTAL] + // backend sampling interface: + + // return true if the backend supports all ops needed by the sampler + // note: call once per sampler + bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft); + + // call after .backend_apply() + void (*backend_accept)( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct ggml_tensor * selected_token); + + // call after .backend_init() + void (*backend_apply)( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct llama_sampler_data * data); + + // called before graph execution to set inputs for the current ubatch + void (*backend_set_input)(struct llama_sampler * smpl); + }; + + struct llama_sampler { + struct llama_sampler_i * iface; + + llama_sampler_context_t ctx; + }; + + // [EXPERIMENTAL] + // attach a sampler to the context + // note: prefer initializing the context with llama_context_params.samplers when possible + LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl); + + // mirror of llama_sampler_i: + LLAMA_API struct llama_sampler * llama_sampler_init ( struct llama_sampler_i * iface, llama_sampler_context_t ctx); + LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); + LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); + LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); + LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); + LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); + // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) + LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); + + // llama_sampler_chain + // a type of llama_sampler that can chain multiple samplers one after another + + LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params); + + // important: takes ownership of the sampler object and will free it when llama_sampler_free is called + LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl); + + // return NULL if: + // - the sampler is NULL + // - the sampler is not a llama_sampler_chain + // - the index is out of bounds, unless i == -1 + // - if i == -1, returns the chain itself (can be used to check if the sampler is a chain) + LLAMA_API struct llama_sampler * llama_sampler_chain_get( struct llama_sampler * chain, int32_t i); + + // the total number of samplers in the chain + LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); + + // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed + LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i); + + // available samplers: + + LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void); + + /// seed == LLAMA_DEFAULT_SEED to use a random seed. + LLAMA_API struct llama_sampler * llama_sampler_init_dist(uint32_t seed); + + /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// Setting k <= 0 makes this a noop + LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); + + /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep); + + /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 + LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep); + + /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep); + + /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf + LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); + + /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. + LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent); + + /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 + LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed); + + /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 + LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n); + + /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API struct llama_sampler * llama_sampler_init_mirostat( + int32_t n_vocab, + uint32_t seed, + float tau, + float eta, + int32_t m); + + /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2( + uint32_t seed, + float tau, + float eta); + + /// @details Initializes a GBNF grammar, see grammars/README.md for details. + /// @param vocab The vocabulary that this grammar will be used with. + /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. + /// @param grammar_root The name of the start symbol for the grammar. + LLAMA_API struct llama_sampler * llama_sampler_init_grammar( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root); + + DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root, + const char ** trigger_words, + size_t num_trigger_words, + const llama_token * trigger_tokens, + size_t num_trigger_tokens), + "use llama_sampler_init_grammar_lazy_patterns instead"); + + + /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 + /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. + /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. + LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root, + const char ** trigger_patterns, + size_t num_trigger_patterns, + const llama_token * trigger_tokens, + size_t num_trigger_tokens); + + + /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. + LLAMA_API struct llama_sampler * llama_sampler_init_penalties( + int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) + float penalty_repeat, // 1.0 = disabled + float penalty_freq, // 0.0 = disabled + float penalty_present); // 0.0 = disabled + + /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 + LLAMA_API struct llama_sampler * llama_sampler_init_dry( + const struct llama_vocab * vocab, + int32_t n_ctx_train, + float dry_multiplier, + float dry_base, + int32_t dry_allowed_length, + int32_t dry_penalty_last_n, + const char ** seq_breakers, + size_t num_breakers); + + /// adaptive-p: select tokens near a configurable target probability over time. + /// + /// the adaptive-p sampler transforms the token probability distribution to favor tokens + /// that fall near a user-configurable probability target. + /// + /// internally, the sampler maintains an exponential moving average of the *ORIGINAL* + /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an + /// adapted target probability at each sampling step, thus maintaining the desired target + /// probability over time. + /// + /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last + /// in the sampler chain (like mirostat, dist, greedy). + /// + /// only mild truncation before this sampler is recommended. we suggest applying min-p + /// before adaptive-p as the only other active sampler in the chain. + /// + /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) + /// @param decay EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99) + /// @param seed RNG seed + /// + /// ref: https://github.com/ggml-org/llama.cpp/pull/17927 + /// + LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p( + float target, + float decay, + uint32_t seed); + + LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( + int32_t n_vocab, + int32_t n_logit_bias, + const llama_logit_bias * logit_bias); + + // this sampler is meant to be used for fill-in-the-middle infilling + // it's supposed to be used after top_k + top_p sampling + // + // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG + // 2. combine probs of tokens that have the same prefix + // + // example: + // + // - before: + // "hel": 0.5 + // "hell": 0.2 + // "hello": 0.1 + // "dummy": 0.1 + // + // - after: + // "hel": 0.8 + // "dummy": 0.1 + // + // 3. discard non-EOG tokens with low prob + // 4. if no tokens are left -> pick EOT + // + LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab); + + // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise + LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl); + + /// @details Sample and accept a token from the idx-th output of the last evaluation + // + // Shorthand for: + // const auto * logits = llama_get_logits_ith(ctx, idx); + // llama_token_data_array cur_p = { ... init from logits ... }; + // llama_sampler_apply(smpl, &cur_p); + // auto token = cur_p.data[cur_p.selected].id; + // llama_sampler_accept(smpl, token); + // return token; + // Returns the sampled token + LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); + + // TODO: extend in the future + //LLAMA_API void llama_decode_with_sampler(struct llama_context * ctx, struct llama_sampler * smpl, struct llama_batch batch, ...); + + // + // Model split + // + + /// @details Build a split GGUF final path for this chunk. + /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" + // Returns the split_path length. + LLAMA_API int32_t llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int32_t split_no, int32_t split_count); + + /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. + /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" + // Returns the split_prefix length. + LLAMA_API int32_t llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int32_t split_no, int32_t split_count); + + // Print system information + LLAMA_API const char * llama_print_system_info(void); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + // The logger state is global so these functions are NOT thread safe. + LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data); + LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); + + // + // Performance utils + // + // NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements. + // + + struct llama_perf_context_data { + // ms == milliseconds + double t_start_ms; // absolute start time + double t_load_ms; // time needed for loading the model + double t_p_eval_ms; // time needed for processing the prompt + double t_eval_ms; // time needed for generating tokens + + int32_t n_p_eval; // number of prompt tokens + int32_t n_eval; // number of generated tokens + int32_t n_reused; // number of times a ggml compute graph had been reused + }; + + struct llama_perf_sampler_data { + double t_sample_ms; // time needed for sampling in ms + + int32_t n_sample; // number of sampled tokens + }; + + LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx); + LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); + LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); + + // NOTE: the following work only with samplers constructed via llama_sampler_chain_init + LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain); + LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); + LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); + + // + // training + // + + // function that returns whether or not a given tensor contains trainable parameters + typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata); + + // always returns true + LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata); + + struct llama_opt_params { + uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0 + + llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters + void * param_filter_ud; // userdata for determining which tensors contain trainable parameters + + ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters + void * get_opt_pars_ud; // userdata for calculating optimizer parameters + + enum ggml_opt_optimizer_type optimizer_type; + }; + + LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params); + + LLAMA_API void llama_opt_epoch( + struct llama_context * lctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, + ggml_opt_result_t result_eval, + int64_t idata_split, + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval); + +#ifdef __cplusplus +} +#endif + +#endif // LLAMA_H diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/mtmd-helper.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/mtmd-helper.h new file mode 100644 index 00000000..164b7c66 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/mtmd-helper.h @@ -0,0 +1,176 @@ +#ifndef MTMD_HELPER_H +#define MTMD_HELPER_H + +#include "ggml.h" +#include "llama.h" +#include "mtmd.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// +// libmtmd helper functions +// +// Please note that these helpers are not guaranteed to be stable. +// BREAKING CHANGES are expected. +// + +struct mtmd_helper_video; +typedef struct mtmd_helper_video mtmd_helper_video; + +// Set callback for all future logging events. +// If this is not called, or NULL is supplied, everything is output on stderr. +// Note: this also call mtmd_log_set() internally +MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data); + +// Returns true if this build includes video support (MTMD_VIDEO was ON at compile time). +MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx); + +struct mtmd_helper_bitmap_wrapper { + mtmd_bitmap * bitmap; + mtmd_helper_video * video_ctx; +}; + +// helper function to construct a mtmd_bitmap from a file +// it calls mtmd_helper_bitmap_init_from_buf() internally +// returns nullptr on failure +// this function is thread-safe +MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); + +// helper function to construct a mtmd_bitmap from a buffer containing a file +// supported formats: +// image: formats supported by stb_image: jpg, png, bmp, gif, etc. +// audio: formats supported by miniaudio: wav, mp3, flac +// note: +// - for now, video input is only supported via C++ helper functions +// - audio files will be auto-detected based on magic bytes +// - output bitmap will have FNV hash as the ID +// returns nullptr on failure +// this function is thread-safe +MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); + +// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache +MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); + +// helper to count the total position of tokens from a list of chunks, useful to keep track of n_past +// normally, n_pos is equal to n_tokens, but for M-RoPE it is different +MTMD_API llama_pos mtmd_helper_get_n_pos(const mtmd_input_chunks * chunks); + +// helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE +// out_pos must have length == mtmd_helper_get_n_tokens(image) +MTMD_API void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * image, llama_pos pos_0, struct mtmd_decoder_pos * out_pos); + +// helper function that automatically: +// 1. run llama_decode() on text chunks +// 2. run mtmd_encode() on image chunks, then mtmd_get_output_embd() and then llama_decode() +// if any of the mtmd_encode() or llama_decode() calls return non-zero, stop and forward the error +// otherwise, returns 0 on success +// this function is NOT thread-safe +MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunks * chunks, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + bool logits_last, + llama_pos * new_n_past); + +// works like mtmd_helper_eval_chunks(), but only for a single chunk +// this function is NOT thread-safe +MTMD_API int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunk * chunk, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + bool logits_last, + llama_pos * new_n_past); + +// helper function to decode an image whose embeddings have already been calculated +// this helper will handle batching and pre/post decoding setup (for ex. gemma 3 requires non-causal attention) +// ret 0 on success, -1 on chunk not being a valid image chunk, 1 on decode failure +MTMD_API int32_t mtmd_helper_decode_image_chunk(mtmd_context * ctx, + struct llama_context * lctx, + const mtmd_input_chunk * chunk, + float * encoded_embd, + llama_pos n_past, + llama_seq_id seq_id, + int32_t n_batch, + llama_pos * new_n_past); + +// +// video input helpers (requires ffmpeg/ffprobe installed on the system) +// the notion of video only exists at the helper level, it is not visible to the core mtmd library +// +// NOTE: this implementation is model-agnostic, it can be used with any vision-capable model +// however, it may not be accurate for some specific models +// (this is expected for now, to keep the implementation simple) +// + +struct mtmd_helper_video_info { + uint32_t width; + uint32_t height; + float fps; // effective fps (fps_target if set, else original video fps) + int32_t n_frames; // estimated total frames at effective fps (-1 if unknown) +}; + +struct mtmd_helper_video_init_params { + float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f + const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH + int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms + // TODO @ngxson : allow "placeholder" bitmap output for counting tokens +}; + +MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void); + +// returns NULL on failure (ffprobe not found, file unreadable, etc.) +MTMD_API mtmd_helper_video * mtmd_helper_video_init( + struct mtmd_context * mctx, + const char * path, + struct mtmd_helper_video_init_params params); + +// Same as mtmd_helper_video_init(), but reads from an in-memory buffer. +// The buffer is copied internally; the caller does not need to keep it alive. +// Note: pipe input is not seekable, so seeking will use output-side seeking +// (ffmpeg decodes and discards frames up to the target position). +MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( + struct mtmd_context * mctx, + const unsigned char * buf, size_t len, + struct mtmd_helper_video_init_params params); +MTMD_API void mtmd_helper_video_free(mtmd_helper_video * ctx); +MTMD_API struct mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx); + +// Read the next item from the video stream; exactly one of out_bitmap or out_text is set per call. +// *out_bitmap - heap-allocated; caller must free with mtmd_bitmap_free() +// *out_text - heap-allocated (always via strdup/malloc); caller must free with free() +// returns 0 on success, -1 on EOF, -2 on error +MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, + mtmd_bitmap ** out_bitmap, + char ** out_text); + +#ifdef __cplusplus +} // extern "C" +#endif + +// +// C++ wrappers +// + +#ifdef __cplusplus +namespace mtmd_helper { + +// video-related C++ wrappers +struct mtmd_helper_video_deleter { + void operator()(mtmd_helper_video * val) { mtmd_helper_video_free(val); } +}; +using video_ptr = std::unique_ptr; + +} // namespace mtmd_helper +#endif + +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/mtmd.h b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/mtmd.h new file mode 100644 index 00000000..a76a6ec2 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Headers/mtmd.h @@ -0,0 +1,387 @@ +#ifndef MTMD_H +#define MTMD_H + +#include "ggml.h" +#include "llama.h" + +#include +#include +#include + +#ifdef __cplusplus +#include +#include +#include +#include +#include +#endif + +/** + * libmtmd: A library for multimodal support in llama.cpp. + * + * WARNING: This API is experimental and subject to many BREAKING CHANGES. + * Issues related to API usage may receive lower priority support. + * + * For the usage, see an example in mtmd-cli.cpp + * + * For contributors: + * - Make sure the C API is aligned with the libllama C API (as in llama.h) + * - Do not include model name (e.g., qwen, gemma) in the API, use generic terms instead + * - Keep the API minimal, do not expose internal details unless necessary + * + * IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated. + * We encourage human contributors to ensure the quality and reliability of the codebase. + */ + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define MTMD_API __declspec(dllexport) +# else +# define MTMD_API __declspec(dllimport) +# endif +# else +# define MTMD_API __attribute__ ((visibility ("default"))) +# endif +#else +# define MTMD_API +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +enum mtmd_input_chunk_type { + MTMD_INPUT_CHUNK_TYPE_TEXT, + MTMD_INPUT_CHUNK_TYPE_IMAGE, + MTMD_INPUT_CHUNK_TYPE_AUDIO, +}; + +// opaque types +struct mtmd_context; +struct mtmd_bitmap; +struct mtmd_image_tokens; +struct mtmd_input_chunk; +struct mtmd_input_chunks; + +struct mtmd_input_text { + const char * text; + bool add_special; + bool parse_special; +}; + +// +// C API +// + +typedef struct mtmd_context mtmd_context; +typedef struct mtmd_bitmap mtmd_bitmap; +typedef struct mtmd_image_tokens mtmd_image_tokens; +typedef struct mtmd_input_chunk mtmd_input_chunk; +typedef struct mtmd_input_chunks mtmd_input_chunks; +typedef struct mtmd_input_text mtmd_input_text; + +struct mtmd_context_params { + bool use_gpu; + bool print_timings; + int n_threads; + const char * image_marker; // deprecated, use media_marker instead + const char * media_marker; + enum llama_flash_attn_type flash_attn_type; + bool warmup; // whether to run a warmup encode pass after initialization + + // limit number of image tokens, only for vision models with dynamic resolution + int image_min_tokens; // minimum number of tokens for image input (default: read from metadata) + int image_max_tokens; // maximum number of tokens for image input (default: read from metadata) + + // callback function passed over to mtmd proper + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; +}; + +MTMD_API const char * mtmd_default_marker(void); + +MTMD_API struct mtmd_context_params mtmd_context_params_default(void); + +// initialize the mtmd context +// return nullptr on failure +MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname, + const struct llama_model * text_model, + const struct mtmd_context_params ctx_params); + +MTMD_API void mtmd_free(mtmd_context * ctx); + +// whether we need to set non-causal mask before llama_decode +// if chunk is nullptr, we assume the default case where chunk is an image chunk +MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk); + +// whether the current model use M-RoPE for llama_decode +MTMD_API bool mtmd_decode_use_mrope(const mtmd_context * ctx); + +// whether the current model supports vision input +MTMD_API bool mtmd_support_vision(const mtmd_context * ctx); + +// whether the current model supports audio input +MTMD_API bool mtmd_support_audio(const mtmd_context * ctx); + +// get audio sample rate in Hz, for example 16000 for Whisper +// return -1 if audio is not supported +MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); + +// get the current marker string +MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); + +// mtmd_bitmap +// +// if bitmap is image: +// length of data must be nx * ny * 3 +// the data is in RGBRGBRGB... format +// note: some video-capable models (i.e. qwen-vl) can merge consecutive bitmaps +// into one chunk, mtmd_tokenize() will automatically handle this +// if bitmap is audio: +// length of data must be n_samples * sizeof(float) +// the data is in float format (PCM F32) +// +// if data == nullptr: +// the bitmap is considered "empty", and will be treated as a placeholder for counting tokens +// you can pass the bitmap via mtmd_tokenize(), then call mtmd_*_get_n_tokens() to count the tokens +// note: passing a placeholder bitmap to mtmd_encode() will return an error +MTMD_API mtmd_bitmap * mtmd_bitmap_init (uint32_t nx, uint32_t ny, const unsigned char * data); +MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); +MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap); +MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap); +MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); +MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap); +MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); +MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); +// bitmap ID is optional, but useful for KV cache tracking +// these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data() +MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap); +MTMD_API void mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id); + +// mtmd_bitmap lazy +// +// this is a special bitmap that: +// - does not hold the actual data +// - can be expanded into one or more chunks (either media to text chunks) +// user must provide a callback to fill in the data when mtmd_tokenize() is called +// this is useful for large video inputs: +// - allow reading video frame by frame, without loading the entire video into memory +// - allow tracking the whole video with a single ID (for example, the file hash) + +// set (*out_bitmap) to non-nullptr to emit a bitmap chunk; it will be freed automatically +// set (*out_text) to non-nullptr to emit a text chunk; it must be heap-allocated, null-terminated and will be freed automatically +// either out_bitmap or out_text can be set, but not both +// out_bitmap cannot be another lazy bitmap (no nested lazy allowed) +// return value: +// 0 on success +// -1 on EOF (signal to mtmd_tokenize to move on) +// -2 on error (signal to mtmd_tokenize to abort) +typedef int(* mtmd_bitmap_lazy_callback)( + size_t chunk_idx, + void * user_data, + mtmd_bitmap ** out_bitmap, + char ** out_text); + +MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx, + const char * id, // usually set to file hash + void * user_data, + mtmd_bitmap_lazy_callback callback); + +// mtmd_input_chunks +// +// this is simply a list of mtmd_input_chunk +// the elements can only be populated via mtmd_tokenize() +MTMD_API mtmd_input_chunks * mtmd_input_chunks_init(void); +MTMD_API size_t mtmd_input_chunks_size(const mtmd_input_chunks * chunks); +MTMD_API const mtmd_input_chunk * mtmd_input_chunks_get (const mtmd_input_chunks * chunks, size_t idx); +MTMD_API void mtmd_input_chunks_free(mtmd_input_chunks * chunks); + +// mtmd_input_chunk +// +// the instance will be constructed via mtmd_tokenize() +// it will be freed along with mtmd_input_chunks +MTMD_API enum mtmd_input_chunk_type mtmd_input_chunk_get_type (const mtmd_input_chunk * chunk); +MTMD_API const llama_token * mtmd_input_chunk_get_tokens_text (const mtmd_input_chunk * chunk, size_t * n_tokens_output); +MTMD_API const mtmd_image_tokens * mtmd_input_chunk_get_tokens_image(const mtmd_input_chunk * chunk); +MTMD_API size_t mtmd_input_chunk_get_n_tokens (const mtmd_input_chunk * chunk); +// returns nullptr for ID on text chunk +MTMD_API const char * mtmd_input_chunk_get_id (const mtmd_input_chunk * chunk); +// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise) +MTMD_API llama_pos mtmd_input_chunk_get_n_pos (const mtmd_input_chunk * chunk); + +// in case you want to use custom logic to handle the chunk (i.e. KV cache management) +// you can move the chunk ownership to your own code by copying it +// remember to free the chunk when you are done with it +MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk); +MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk); + + +// mtmd_image_tokens +// +// the instance will be constructed via mtmd_tokenize() +// it will be freed along with mtmd_input_chunk +MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); // TODO: deprecate +MTMD_API const char * mtmd_image_tokens_get_id (const mtmd_image_tokens * image_tokens); // TODO: deprecate +// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise) +MTMD_API llama_pos mtmd_image_tokens_get_n_pos (const mtmd_image_tokens * image_tokens); // TODO: deprecate + +DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens), + "use mtmd_image_tokens_get_decoder_pos() instead"); +DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens), + "use mtmd_image_tokens_get_decoder_pos() instead"); + +struct mtmd_decoder_pos { + uint32_t t; + uint32_t x; + uint32_t y; + uint32_t z; // unused for now, reserved for future use +}; +// get position for decoder attention, to be used by M-RoPE models +// i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 +// pos_0 is the absolute position of the first token +// return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position) +MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i); + +// tokenize an input text prompt and a list of bitmaps (images/audio) +// the prompt must have the input image marker (default: "<__media__>") in it +// the default marker is defined by mtmd_default_marker() +// the marker will be replaced with the image/audio chunk +// for example: +// "here is an image: <__media__>\ndescribe it in detail." +// this will gives 3 chunks: +// 1. "here is an image: " +// 2. (image/audio tokens) +// 3. "\ndescribe it in detail." +// number of bitmaps must be equal to the number of markers in the prompt +// this function is thread-safe (shared ctx) +// return values: +// 0 on success +// 1 on number of bitmaps not matching the number of markers +// 2 on image preprocessing error +MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, + mtmd_input_chunks * output, + const mtmd_input_text * text, + const mtmd_bitmap ** bitmaps, + size_t n_bitmaps); + +// returns 0 on success +// TODO: deprecate +MTMD_API int32_t mtmd_encode(mtmd_context * ctx, + const mtmd_image_tokens * image_tokens); + +// returns 0 on success +MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx, + const mtmd_input_chunk * chunk); + +// get output embeddings from the last encode pass +// the reading size (in bytes) is equal to: +// llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float) +MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx); + +// Set callback for all future logging events. +// If this is not called, or NULL is supplied, everything is output on stderr. +MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); + +// EXPERIMENTAL API to get mmproj's capabilities without initializing the full context +// This is only intended to be used by llama-server, breaking changes is expected +struct mtmd_caps { + bool inp_vision; + bool inp_audio; +}; +MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname); + +///////////////////////////////////////// + +// test function, to be used in test-mtmd-c-api.c +MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +// Get memory usage of the current model in bytes, per backend device +// Note: this is an unstable API, used internally by fit_params; it WILL be removed or changed without deprecation +#ifdef __cplusplus +MTMD_API std::map mtmd_get_memory_usage( + const char * mmproj_fname, + struct mtmd_context_params ctx_params); +#endif + +// +// C++ wrappers +// + +#ifdef __cplusplus + +namespace mtmd { + +struct mtmd_context_deleter { + void operator()(mtmd_context * val) { mtmd_free(val); } +}; +using context_ptr = std::unique_ptr; + +struct mtmd_bitmap_deleter { + void operator()(mtmd_bitmap * val) { mtmd_bitmap_free(val); } +}; +using bitmap_ptr = std::unique_ptr; + +struct mtmd_input_chunks_deleter { + void operator()(mtmd_input_chunks * val) { mtmd_input_chunks_free(val); } +}; +using input_chunks_ptr = std::unique_ptr; + +struct mtmd_input_chunk_deleter { + void operator()(mtmd_input_chunk * val) { mtmd_input_chunk_free(val); } +}; +using input_chunk_ptr = std::unique_ptr; + +struct bitmap { + bitmap_ptr ptr; + bitmap() : ptr(nullptr) {} + bitmap(mtmd_bitmap * bitmap) : ptr(bitmap) {} + bitmap(bitmap && other) noexcept : ptr(std::move(other.ptr)) {} + bitmap(uint32_t nx, uint32_t ny, const unsigned char * data) { + ptr.reset(mtmd_bitmap_init(nx, ny, data)); + } + ~bitmap() = default; + uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); } + uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); } + const unsigned char * data() const { return mtmd_bitmap_get_data(ptr.get()); } + size_t n_bytes() const { return mtmd_bitmap_get_n_bytes(ptr.get()); } + std::string id() const { return mtmd_bitmap_get_id(ptr.get()); } + void set_id(const char * id) const { mtmd_bitmap_set_id(ptr.get(), id); } +}; + +struct bitmaps { + std::vector entries; + ~bitmaps() = default; + // return list of pointers to mtmd_bitmap + // example: + // auto bitmaps_c_ptr = bitmaps.c_ptr(); + // int32_t res = mtmd_tokenize(... bitmaps_c_ptr.data(), bitmaps_c_ptr.size()); + std::vector c_ptr() { + std::vector res(entries.size()); + for (size_t i = 0; i < entries.size(); i++) { + res[i] = entries[i].ptr.get(); + } + return res; + } +}; + +struct input_chunks { + input_chunks_ptr ptr; + input_chunks() = default; + input_chunks(mtmd_input_chunks * chunks) : ptr(chunks) {} + ~input_chunks() = default; + size_t size() const { return mtmd_input_chunks_size(ptr.get()); } + const mtmd_input_chunk * operator[](size_t idx) const { + return mtmd_input_chunks_get(ptr.get(), idx); + } +}; + +} // namespace mtmd + +#endif + +#endif diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Modules/module.modulemap b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Modules/module.modulemap new file mode 100644 index 00000000..4d610a57 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Modules/module.modulemap @@ -0,0 +1,5 @@ +framework module Llama { + umbrella header "llama.h" + export * + module * { export * } +} diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Resources/Info.plist b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Resources/Info.plist new file mode 100644 index 00000000..7092b308 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/Resources/Info.plist @@ -0,0 +1,16 @@ + + + + + CFBundleDevelopmentRegionen + CFBundleExecutableLlama + CFBundleIdentifierorg.ggml.llama + CFBundleInfoDictionaryVersion6.0 + CFBundleNameLlama + CFBundlePackageTypeFMWK + CFBundleShortVersionString0.9.0 + CFBundleVersion1 + MinimumOSVersion12.0 + + diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/_CodeSignature/CodeResources b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/_CodeSignature/CodeResources new file mode 100644 index 00000000..0a907c60 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/_CodeSignature/CodeResources @@ -0,0 +1,205 @@ + + + + + files + + Resources/Info.plist + + 3PSjo1YtkVnoPlJFxHZLxILHaqM= + + + files2 + + Headers/ggml-alloc.h + + hash2 + + lOTNBpuTE7LOs12s7JAZgeC7R42LsxA1txJr4JGZjCM= + + + Headers/ggml-backend.h + + hash2 + + piDoFbQ6RMxy1fIWYpo6kZgDNbYbw362stCBM2jDcE8= + + + Headers/ggml-cpu.h + + hash2 + + Gq/pflduo4wNpXUX+ySSlV0LacGnmYQkgfwGnW2dKO8= + + + Headers/ggml-metal.h + + hash2 + + Mi82zTDz6eeq17W5vGMHgBL9C3cGrE4k9XIbYE89iYA= + + + Headers/ggml-opt.h + + hash2 + + NYbeG8ipNLXHIznitpN7BkHo8Um1EiMeZm9n3gc27qI= + + + Headers/ggml.h + + hash2 + + 7fzmpz5ifSIN2K68xtYgkXVbEAYUrX2lMpKlKLGwB1A= + + + Headers/gguf.h + + hash2 + + Ldsnalvs50NDMWCthjJ5RzQxydTBcdRov4YKPNo/vz8= + + + Headers/llama.h + + hash2 + + VrCntN56IKB+D6+KWox7hbhvo6E2c4XcmMr+Gm3HrUY= + + + Headers/mtmd-helper.h + + hash2 + + U+xT7AinBJ5PF493agD+pZ7wGPsLuw8sgAH17wfcCxk= + + + Headers/mtmd.h + + hash2 + + xX/4y6LQ5ZtrHg6Q/snhiHXMUmi65nNvxIuii2TPnPE= + + + Modules/module.modulemap + + hash2 + + YrZrr8fQjLcP/nkdW+LigUSHc1OnNtYwwRm5qRmrFFw= + + + Resources/Info.plist + + hash2 + + 6Mcex1hCuRFVjgwmHADur+Dsg1Zl3uFwuz877kPwBUQ= + + + + rules + + ^Resources/ + + ^Resources/.*\.lproj/ + + optional + + weight + 1000 + + ^Resources/.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Resources/Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS|Library/(Automator|Spotlight|LoginItems))/ + + nested + + weight + 10 + + ^.* + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^Resources/ + + weight + 20 + + ^Resources/.*\.lproj/ + + optional + + weight + 1000 + + ^Resources/.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Resources/Base\.lproj/ + + weight + 1010 + + ^[^/]+$ + + nested + + weight + 10 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/llama b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/llama new file mode 100755 index 00000000..4c85bb85 Binary files /dev/null and b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/A/llama differ diff --git a/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/Current b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/Current new file mode 120000 index 00000000..8c7e5a66 --- /dev/null +++ b/ios/Llama.xcframework/macos-arm64/Llama.framework/Versions/Current @@ -0,0 +1 @@ +A \ No newline at end of file diff --git a/ios/llama_cpp_dart.podspec b/ios/llama_cpp_dart.podspec new file mode 100644 index 00000000..322c9bb3 --- /dev/null +++ b/ios/llama_cpp_dart.podspec @@ -0,0 +1,18 @@ +Pod::Spec.new do |s| + s.name = 'llama_cpp_dart' + s.version = '0.9.0-dev.9' + s.summary = 'Dart / Flutter FFI binding for llama.cpp' + s.description = 'High-level Dart and Flutter bindings for llama.cpp on iOS, macOS, and Android.' + s.homepage = 'https://github.com/netdur/llama_cpp_dart' + s.license = { :type => 'MIT', :file => '../LICENSE' } + s.author = { 'Adel Abdelaty' => 'netdur@gmail.com' } + s.source = { :path => '.' } + + s.ios.deployment_target = '14.0' + s.osx.deployment_target = '11.0' + + s.vendored_frameworks = 'Llama.xcframework' + + s.frameworks = 'Metal', 'MetalKit', 'Foundation', 'Accelerate' + s.dependency 'Flutter' +end diff --git a/lib/src/context/context_params.dart b/lib/src/context/context_params.dart index ee912e5c..0a2dab99 100644 --- a/lib/src/context/context_params.dart +++ b/lib/src/context/context_params.dart @@ -182,6 +182,24 @@ final class ContextParams { this.nRsSeq = 0, }); + /// Factory preset optimized for memory-constrained mobile devices (iOS / Android). + /// Uses a 1024 token context window and 128 batch size to ensure predictable RAM overhead. + factory ContextParams.mobile({ + int nCtx = 1024, + int nBatch = 128, + int nUbatch = 128, + KvCacheType typeK = KvCacheType.f16, + KvCacheType typeV = KvCacheType.f16, + }) { + return ContextParams( + nCtx: nCtx, + nBatch: nBatch, + nUbatch: nUbatch, + typeK: typeK, + typeV: typeV, + ); + } + ContextParams copyWith({ int? nCtx, int? nBatch, diff --git a/lib/src/ffi/bindings.dart b/lib/src/ffi/bindings.dart index cfaaae8d..cf232921 100644 --- a/lib/src/ffi/bindings.dart +++ b/lib/src/ffi/bindings.dart @@ -3,7 +3,7 @@ // AUTO GENERATED FILE, DO NOT EDIT. // // Generated by `package:ffigen`. -// ignore_for_file: unused_import +// ignore_for_file: unused_import, unused_element, deprecated_member_use_from_same_package import 'dart:ffi' as ffi; /// Raw FFI bindings for llama.cpp (llama.h + mtmd.h). Do not edit by hand. @@ -22,83 +22,71 @@ class LlamaBindings { lookup) : _lookup = lookup; - int renameat( - int arg0, - ffi.Pointer arg1, + int __snprintf_chk( + ffi.Pointer arg0, + int __maxlen, int arg2, - ffi.Pointer arg3, + int arg3, + ffi.Pointer arg4, ) { - return _renameat( + return ___snprintf_chk( arg0, - arg1, + __maxlen, arg2, arg3, + arg4, ); } - late final _renameatPtr = _lookup< + late final ___snprintf_chkPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Int, ffi.Pointer, ffi.Int, - ffi.Pointer)>>('renameat'); - late final _renameat = _renameatPtr.asFunction< - int Function(int, ffi.Pointer, int, ffi.Pointer)>(); + ffi.Int Function(ffi.Pointer, ffi.Size, ffi.Int, ffi.Size, + ffi.Pointer)>>('__snprintf_chk'); + late final ___snprintf_chk = ___snprintf_chkPtr.asFunction< + int Function( + ffi.Pointer, int, int, int, ffi.Pointer)>(); - int renamex_np( + int __sprintf_chk( ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, - ) { - return _renamex_np( - arg0, - arg1, - arg2, - ); - } - - late final _renamex_npPtr = _lookup< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, - ffi.UnsignedInt)>>('renamex_np'); - late final _renamex_np = _renamex_npPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int)>(); - - int renameatx_np( - int arg0, - ffi.Pointer arg1, + int arg1, int arg2, ffi.Pointer arg3, - int arg4, ) { - return _renameatx_np( + return ___sprintf_chk( arg0, arg1, arg2, arg3, - arg4, ); } - late final _renameatx_npPtr = _lookup< + late final ___sprintf_chkPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Int, ffi.Pointer, ffi.Int, - ffi.Pointer, ffi.UnsignedInt)>>('renameatx_np'); - late final _renameatx_np = _renameatx_npPtr.asFunction< - int Function( - int, ffi.Pointer, int, ffi.Pointer, int)>(); + ffi.Int Function(ffi.Pointer, ffi.Int, ffi.Size, + ffi.Pointer)>>('__sprintf_chk'); + late final ___sprintf_chk = ___sprintf_chkPtr.asFunction< + int Function(ffi.Pointer, int, int, ffi.Pointer)>(); - int printf( - ffi.Pointer arg0, + int __srget( + ffi.Pointer arg0, ) { - return _printf( + return ___srget( arg0, ); } - late final _printfPtr = - _lookup)>>( - 'printf'); - late final _printf = - _printfPtr.asFunction)>(); + late final ___srgetPtr = + _lookup)>>( + '__srget'); + late final ___srget = + ___srgetPtr.asFunction)>(); + + late final ffi.Pointer> ___stderrp = + _lookup>('__stderrp'); + + ffi.Pointer get __stderrp => ___stderrp.value; + + set __stderrp(ffi.Pointer value) => ___stderrp.value = value; late final ffi.Pointer> ___stdinp = _lookup>('__stdinp'); @@ -114,12 +102,108 @@ class LlamaBindings { set __stdoutp(ffi.Pointer value) => ___stdoutp.value = value; - late final ffi.Pointer> ___stderrp = - _lookup>('__stderrp'); + int __svfscanf( + ffi.Pointer arg0, + ffi.Pointer arg1, + va_list arg2, + ) { + return ___svfscanf( + arg0, + arg1, + arg2, + ); + } - ffi.Pointer get __stderrp => ___stderrp.value; + late final ___svfscanfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + va_list)>>('__svfscanf'); + late final ___svfscanf = ___svfscanfPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, va_list)>(); - set __stderrp(ffi.Pointer value) => ___stderrp.value = value; + int __swbuf( + int arg0, + ffi.Pointer arg1, + ) { + return ___swbuf( + arg0, + arg1, + ); + } + + late final ___swbufPtr = + _lookup)>>( + '__swbuf'); + late final ___swbuf = + ___swbufPtr.asFunction)>(); + + int __vsnprintf_chk( + ffi.Pointer arg0, + int __maxlen, + int arg2, + int arg3, + ffi.Pointer arg4, + va_list arg5, + ) { + return ___vsnprintf_chk( + arg0, + __maxlen, + arg2, + arg3, + arg4, + arg5, + ); + } + + late final ___vsnprintf_chkPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Size, ffi.Int, ffi.Size, + ffi.Pointer, va_list)>>('__vsnprintf_chk'); + late final ___vsnprintf_chk = ___vsnprintf_chkPtr.asFunction< + int Function(ffi.Pointer, int, int, int, ffi.Pointer, + va_list)>(); + + int __vsprintf_chk( + ffi.Pointer arg0, + int arg1, + int arg2, + ffi.Pointer arg3, + va_list arg4, + ) { + return ___vsprintf_chk( + arg0, + arg1, + arg2, + arg3, + arg4, + ); + } + + late final ___vsprintf_chkPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Int, ffi.Size, + ffi.Pointer, va_list)>>('__vsprintf_chk'); + late final ___vsprintf_chk = ___vsprintf_chkPtr.asFunction< + int Function( + ffi.Pointer, int, int, ffi.Pointer, va_list)>(); + + int asprintf( + ffi.Pointer> arg0, + ffi.Pointer arg1, + ) { + return _asprintf( + arg0, + arg1, + ); + } + + late final _asprintfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer>, + ffi.Pointer)>>('asprintf'); + late final _asprintf = _asprintfPtr.asFunction< + int Function( + ffi.Pointer>, ffi.Pointer)>(); void clearerr( ffi.Pointer arg0, @@ -135,6 +219,50 @@ class LlamaBindings { late final _clearerr = _clearerrPtr.asFunction)>(); + ffi.Pointer ctermid( + ffi.Pointer arg0, + ) { + return _ctermid( + arg0, + ); + } + + late final _ctermidPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer)>>('ctermid'); + late final _ctermid = _ctermidPtr + .asFunction Function(ffi.Pointer)>(); + + ffi.Pointer ctermid_r( + ffi.Pointer arg0, + ) { + return _ctermid_r( + arg0, + ); + } + + late final _ctermid_rPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer)>>('ctermid_r'); + late final _ctermid_r = _ctermid_rPtr + .asFunction Function(ffi.Pointer)>(); + + int dprintf( + int arg0, + ffi.Pointer arg1, + ) { + return _dprintf( + arg0, + arg1, + ); + } + + late final _dprintfPtr = _lookup< + ffi.NativeFunction)>>( + 'dprintf'); + late final _dprintf = + _dprintfPtr.asFunction)>(); + int fclose( ffi.Pointer arg0, ) { @@ -148,6 +276,23 @@ class LlamaBindings { 'fclose'); late final _fclose = _fclosePtr.asFunction)>(); + ffi.Pointer fdopen( + int arg0, + ffi.Pointer arg1, + ) { + return _fdopen( + arg0, + arg1, + ); + } + + late final _fdopenPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Int, ffi.Pointer)>>('fdopen'); + late final _fdopen = _fdopenPtr + .asFunction Function(int, ffi.Pointer)>(); + int feof( ffi.Pointer arg0, ) { @@ -198,11 +343,29 @@ class LlamaBindings { _lookup)>>('fgetc'); late final _fgetc = _fgetcPtr.asFunction)>(); - int fgetpos( + ffi.Pointer fgetln( ffi.Pointer arg0, - ffi.Pointer arg1, + ffi.Pointer __len, ) { - return _fgetpos( + return _fgetln( + arg0, + __len, + ); + } + + late final _fgetlnPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>>('fgetln'); + late final _fgetln = _fgetlnPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); + + int fgetpos( + ffi.Pointer arg0, + ffi.Pointer arg1, + ) { + return _fgetpos( arg0, arg1, ); @@ -234,6 +397,71 @@ class LlamaBindings { ffi.Pointer Function( ffi.Pointer, int, ffi.Pointer)>(); + int fileno( + ffi.Pointer arg0, + ) { + return _fileno( + arg0, + ); + } + + late final _filenoPtr = + _lookup)>>( + 'fileno'); + late final _fileno = _filenoPtr.asFunction)>(); + + void flockfile( + ffi.Pointer arg0, + ) { + return _flockfile( + arg0, + ); + } + + late final _flockfilePtr = + _lookup)>>( + 'flockfile'); + late final _flockfile = + _flockfilePtr.asFunction)>(); + + ffi.Pointer fmemopen( + ffi.Pointer __buf, + int __size, + ffi.Pointer __mode, + ) { + return _fmemopen( + __buf, + __size, + __mode, + ); + } + + late final _fmemopenPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, ffi.Size, + ffi.Pointer)>>('fmemopen'); + late final _fmemopen = _fmemopenPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, int, ffi.Pointer)>(); + + ffi.Pointer fmtcheck( + ffi.Pointer arg0, + ffi.Pointer arg1, + ) { + return _fmtcheck( + arg0, + arg1, + ); + } + + late final _fmtcheckPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>>('fmtcheck'); + late final _fmtcheck = _fmtcheckPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer fopen( ffi.Pointer __filename, ffi.Pointer __mode, @@ -269,6 +497,19 @@ class LlamaBindings { late final _fprintf = _fprintfPtr .asFunction, ffi.Pointer)>(); + int fpurge( + ffi.Pointer arg0, + ) { + return _fpurge( + arg0, + ); + } + + late final _fpurgePtr = + _lookup)>>( + 'fpurge'); + late final _fpurge = _fpurgePtr.asFunction)>(); + int fputc( int arg0, ffi.Pointer arg1, @@ -356,6 +597,24 @@ class LlamaBindings { late final _fseek = _fseekPtr.asFunction, int, int)>(); + int fseeko( + ffi.Pointer __stream, + int __offset, + int __whence, + ) { + return _fseeko( + __stream, + __offset, + __whence, + ); + } + + late final _fseekoPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, off_t, ffi.Int)>>('fseeko'); + late final _fseeko = + _fseekoPtr.asFunction, int, int)>(); + int fsetpos( ffi.Pointer arg0, ffi.Pointer arg1, @@ -385,6831 +644,6604 @@ class LlamaBindings { 'ftell'); late final _ftell = _ftellPtr.asFunction)>(); - int getc( - ffi.Pointer arg0, + int ftello( + ffi.Pointer __stream, ) { - return _getc( - arg0, + return _ftello( + __stream, ); } - late final _getcPtr = - _lookup)>>('getc'); - late final _getc = _getcPtr.asFunction)>(); - - int getchar() { - return _getchar(); - } - - late final _getcharPtr = - _lookup>('getchar'); - late final _getchar = _getcharPtr.asFunction(); + late final _ftelloPtr = + _lookup)>>('ftello'); + late final _ftello = _ftelloPtr.asFunction)>(); - ffi.Pointer gets( - ffi.Pointer arg0, + int ftrylockfile( + ffi.Pointer arg0, ) { - return _gets( + return _ftrylockfile( arg0, ); } - late final _getsPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer)>>('gets'); - late final _gets = _getsPtr - .asFunction Function(ffi.Pointer)>(); + late final _ftrylockfilePtr = + _lookup)>>( + 'ftrylockfile'); + late final _ftrylockfile = + _ftrylockfilePtr.asFunction)>(); - void perror( - ffi.Pointer arg0, + void funlockfile( + ffi.Pointer arg0, ) { - return _perror( + return _funlockfile( arg0, ); } - late final _perrorPtr = - _lookup)>>( - 'perror'); - late final _perror = - _perrorPtr.asFunction)>(); + late final _funlockfilePtr = + _lookup)>>( + 'funlockfile'); + late final _funlockfile = + _funlockfilePtr.asFunction)>(); - int putc( - int arg0, - ffi.Pointer arg1, + ffi.Pointer funopen( + ffi.Pointer arg0, + ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer, ffi.Int)>> + arg1, + ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer, ffi.Int)>> + arg2, + ffi.Pointer< + ffi.NativeFunction< + fpos_t Function(ffi.Pointer, fpos_t, ffi.Int)>> + arg3, + ffi.Pointer)>> + arg4, ) { - return _putc( + return _funopen( arg0, arg1, + arg2, + arg3, + arg4, ); } - late final _putcPtr = - _lookup)>>( - 'putc'); - late final _putc = - _putcPtr.asFunction)>(); + late final _funopenPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, + ffi.Pointer, ffi.Int)>>, + ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, + ffi.Pointer, ffi.Int)>>, + ffi.Pointer< + ffi.NativeFunction< + fpos_t Function(ffi.Pointer, fpos_t, ffi.Int)>>, + ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer)>>)>>('funopen'); + late final _funopen = _funopenPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer, ffi.Int)>>, + ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer, ffi.Int)>>, + ffi.Pointer< + ffi.NativeFunction< + fpos_t Function(ffi.Pointer, fpos_t, ffi.Int)>>, + ffi.Pointer< + ffi.NativeFunction)>>)>(); - int putchar( - int arg0, + int getc( + ffi.Pointer arg0, ) { - return _putchar( + return _getc( arg0, ); } - late final _putcharPtr = - _lookup>('putchar'); - late final _putchar = _putcharPtr.asFunction(); + late final _getcPtr = + _lookup)>>('getc'); + late final _getc = _getcPtr.asFunction)>(); - int puts( - ffi.Pointer arg0, + int getc_unlocked( + ffi.Pointer arg0, ) { - return _puts( + return _getc_unlocked( arg0, ); } - late final _putsPtr = - _lookup)>>( - 'puts'); - late final _puts = _putsPtr.asFunction)>(); + late final _getc_unlockedPtr = + _lookup)>>( + 'getc_unlocked'); + late final _getc_unlocked = + _getc_unlockedPtr.asFunction)>(); - int remove( - ffi.Pointer arg0, - ) { - return _remove( - arg0, - ); + int getchar() { + return _getchar(); } - late final _removePtr = - _lookup)>>( - 'remove'); - late final _remove = - _removePtr.asFunction)>(); + late final _getcharPtr = + _lookup>('getchar'); + late final _getchar = _getcharPtr.asFunction(); - int rename( - ffi.Pointer __old, - ffi.Pointer __new, + int getchar_unlocked() { + return _getchar_unlocked(); + } + + late final _getchar_unlockedPtr = + _lookup>('getchar_unlocked'); + late final _getchar_unlocked = + _getchar_unlockedPtr.asFunction(); + + int getdelim( + ffi.Pointer> __linep, + ffi.Pointer __linecapp, + int __delimiter, + ffi.Pointer __stream, ) { - return _rename( - __old, - __new, + return _getdelim( + __linep, + __linecapp, + __delimiter, + __stream, ); } - late final _renamePtr = _lookup< + late final _getdelimPtr = _lookup< ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer)>>('rename'); - late final _rename = _renamePtr - .asFunction, ffi.Pointer)>(); + ssize_t Function(ffi.Pointer>, + ffi.Pointer, ffi.Int, ffi.Pointer)>>('getdelim'); + late final _getdelim = _getdelimPtr.asFunction< + int Function(ffi.Pointer>, ffi.Pointer, + int, ffi.Pointer)>(); - void rewind( - ffi.Pointer arg0, + int getline( + ffi.Pointer> __linep, + ffi.Pointer __linecapp, + ffi.Pointer __stream, ) { - return _rewind( - arg0, + return _getline( + __linep, + __linecapp, + __stream, ); } - late final _rewindPtr = - _lookup)>>( - 'rewind'); - late final _rewind = - _rewindPtr.asFunction)>(); + late final _getlinePtr = _lookup< + ffi.NativeFunction< + ssize_t Function(ffi.Pointer>, + ffi.Pointer, ffi.Pointer)>>('getline'); + late final _getline = _getlinePtr.asFunction< + int Function(ffi.Pointer>, ffi.Pointer, + ffi.Pointer)>(); - int scanf( + @Deprecated( + 'This function is provided for compatibility reasons only. Due to security concerns inherent in the design of gets(3), it is highly recommended that you use fgets(3) instead.') + ffi.Pointer gets( ffi.Pointer arg0, ) { - return _scanf( + return _gets( arg0, ); } - late final _scanfPtr = - _lookup)>>( - 'scanf'); - late final _scanf = - _scanfPtr.asFunction)>(); + late final _getsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer)>>('gets'); + late final _gets = _getsPtr + .asFunction Function(ffi.Pointer)>(); - void setbuf( + int getw( ffi.Pointer arg0, - ffi.Pointer arg1, ) { - return _setbuf( + return _getw( arg0, - arg1, ); } - late final _setbufPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, ffi.Pointer)>>('setbuf'); - late final _setbuf = _setbufPtr - .asFunction, ffi.Pointer)>(); + late final _getwPtr = + _lookup)>>('getw'); + late final _getw = _getwPtr.asFunction)>(); - int setvbuf( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, - int __size, + void ggml_abort( + ffi.Pointer file, + int line, + ffi.Pointer fmt, ) { - return _setvbuf( - arg0, - arg1, - arg2, - __size, + return _ggml_abort( + file, + line, + fmt, ); } - late final _setvbufPtr = _lookup< + late final _ggml_abortPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, ffi.Int, - ffi.Size)>>('setvbuf'); - late final _setvbuf = _setvbufPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Void Function(ffi.Pointer, ffi.Int, + ffi.Pointer)>>('ggml_abort'); + late final _ggml_abort = _ggml_abortPtr.asFunction< + void Function(ffi.Pointer, int, ffi.Pointer)>(); - int sprintf( - ffi.Pointer arg0, - ffi.Pointer arg1, + ffi.Pointer ggml_abs( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _sprintf( - arg0, - arg1, + return _ggml_abs( + ctx, + a, ); } - late final _sprintfPtr = _lookup< + late final _ggml_absPtr = _lookup< ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer)>>('sprintf'); - late final _sprintf = _sprintfPtr - .asFunction, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_abs'); + late final _ggml_abs = _ggml_absPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - int sscanf( - ffi.Pointer arg0, - ffi.Pointer arg1, + ffi.Pointer ggml_abs_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _sscanf( - arg0, - arg1, + return _ggml_abs_inplace( + ctx, + a, ); } - late final _sscanfPtr = _lookup< + late final _ggml_abs_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer)>>('sscanf'); - late final _sscanf = _sscanfPtr - .asFunction, ffi.Pointer)>(); - - ffi.Pointer tmpfile() { - return _tmpfile(); - } - - late final _tmpfilePtr = - _lookup Function()>>('tmpfile'); - late final _tmpfile = _tmpfilePtr.asFunction Function()>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_abs_inplace'); + late final _ggml_abs_inplace = _ggml_abs_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer tmpnam( - ffi.Pointer arg0, + ffi.Pointer ggml_acc( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int nb1, + int nb2, + int nb3, + int offset, ) { - return _tmpnam( - arg0, + return _ggml_acc( + ctx, + a, + b, + nb1, + nb2, + nb3, + offset, ); } - late final _tmpnamPtr = _lookup< + late final _ggml_accPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer)>>('tmpnam'); - late final _tmpnam = _tmpnamPtr - .asFunction Function(ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_acc'); + late final _ggml_acc = _ggml_accPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int)>(); - int ungetc( - int arg0, - ffi.Pointer arg1, + ffi.Pointer ggml_acc_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int nb1, + int nb2, + int nb3, + int offset, ) { - return _ungetc( - arg0, - arg1, + return _ggml_acc_inplace( + ctx, + a, + b, + nb1, + nb2, + nb3, + offset, ); } - late final _ungetcPtr = - _lookup)>>( - 'ungetc'); - late final _ungetc = - _ungetcPtr.asFunction)>(); - - int vfprintf( - ffi.Pointer arg0, - ffi.Pointer arg1, - va_list arg2, - ) { - return _vfprintf( - arg0, - arg1, - arg2, - ); - } - - late final _vfprintfPtr = _lookup< + late final _ggml_acc_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer, va_list)>>('vfprintf'); - late final _vfprintf = _vfprintfPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, va_list)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_acc_inplace'); + late final _ggml_acc_inplace = _ggml_acc_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int)>(); - int vprintf( - ffi.Pointer arg0, - va_list arg1, + ffi.Pointer ggml_add( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _vprintf( - arg0, - arg1, + return _ggml_add( + ctx, + a, + b, ); } - late final _vprintfPtr = _lookup< - ffi.NativeFunction, va_list)>>( - 'vprintf'); - late final _vprintf = - _vprintfPtr.asFunction, va_list)>(); + late final _ggml_addPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>>('ggml_add'); + late final _ggml_add = _ggml_addPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - int vsprintf( - ffi.Pointer arg0, - ffi.Pointer arg1, - va_list arg2, + @Deprecated('use ggml_add instead') + ffi.Pointer ggml_add1( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _vsprintf( - arg0, - arg1, - arg2, + return _ggml_add1( + ctx, + a, + b, ); } - late final _vsprintfPtr = _lookup< + late final _ggml_add1Ptr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, - va_list)>>('vsprintf'); - late final _vsprintf = _vsprintfPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, va_list)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_add1'); + late final _ggml_add1 = _ggml_add1Ptr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ctermid( - ffi.Pointer arg0, + @Deprecated('use ggml_add_inplace instead') + ffi.Pointer ggml_add1_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ctermid( - arg0, + return _ggml_add1_inplace( + ctx, + a, + b, ); } - late final _ctermidPtr = _lookup< + late final _ggml_add1_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer)>>('ctermid'); - late final _ctermid = _ctermidPtr - .asFunction Function(ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_add1_inplace'); + late final _ggml_add1_inplace = _ggml_add1_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer fdopen( - int arg0, - ffi.Pointer arg1, + ffi.Pointer ggml_add_cast( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ggml_type type, ) { - return _fdopen( - arg0, - arg1, + return _ggml_add_cast( + ctx, + a, + b, + type.value, ); } - late final _fdopenPtr = _lookup< + late final _ggml_add_castPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Int, ffi.Pointer)>>('fdopen'); - late final _fdopen = _fdopenPtr - .asFunction Function(int, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.UnsignedInt)>>('ggml_add_cast'); + late final _ggml_add_cast = _ggml_add_castPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int)>(); - int fileno( - ffi.Pointer arg0, + ffi.Pointer ggml_add_id( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer ids, ) { - return _fileno( - arg0, + return _ggml_add_id( + ctx, + a, + b, + ids, ); } - late final _filenoPtr = - _lookup)>>( - 'fileno'); - late final _fileno = _filenoPtr.asFunction)>(); + late final _ggml_add_idPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_add_id'); + late final _ggml_add_id = _ggml_add_idPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - int pclose( - ffi.Pointer arg0, + ffi.Pointer ggml_add_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _pclose( - arg0, + return _ggml_add_inplace( + ctx, + a, + b, ); } - late final _pclosePtr = - _lookup)>>( - 'pclose'); - late final _pclose = _pclosePtr.asFunction)>(); + late final _ggml_add_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_add_inplace'); + late final _ggml_add_inplace = _ggml_add_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer popen( - ffi.Pointer arg0, - ffi.Pointer arg1, + ffi.Pointer ggml_add_rel_pos( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer pw, + ffi.Pointer ph, ) { - return _popen( - arg0, - arg1, + return _ggml_add_rel_pos( + ctx, + a, + pw, + ph, ); } - late final _popenPtr = _lookup< + late final _ggml_add_rel_posPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>>('popen'); - late final _popen = _popenPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_add_rel_pos'); + late final _ggml_add_rel_pos = _ggml_add_rel_posPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - int __srget( - ffi.Pointer arg0, + ffi.Pointer ggml_add_rel_pos_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer pw, + ffi.Pointer ph, ) { - return ___srget( - arg0, + return _ggml_add_rel_pos_inplace( + ctx, + a, + pw, + ph, ); } - late final ___srgetPtr = - _lookup)>>( - '__srget'); - late final ___srget = - ___srgetPtr.asFunction)>(); + late final _ggml_add_rel_pos_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_add_rel_pos_inplace'); + late final _ggml_add_rel_pos_inplace = + _ggml_add_rel_pos_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - int __svfscanf( - ffi.Pointer arg0, - ffi.Pointer arg1, - va_list arg2, + ffi.Pointer ggml_arange( + ffi.Pointer ctx, + double start, + double stop, + double step, ) { - return ___svfscanf( - arg0, - arg1, - arg2, + return _ggml_arange( + ctx, + start, + stop, + step, ); } - late final ___svfscanfPtr = _lookup< + late final _ggml_arangePtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, - va_list)>>('__svfscanf'); - late final ___svfscanf = ___svfscanfPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, va_list)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Float, ffi.Float, ffi.Float)>>('ggml_arange'); + late final _ggml_arange = _ggml_arangePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, double, double, double)>(); - int __swbuf( - int arg0, - ffi.Pointer arg1, + bool ggml_are_same_shape( + ffi.Pointer t0, + ffi.Pointer t1, ) { - return ___swbuf( - arg0, - arg1, + return _ggml_are_same_shape( + t0, + t1, ); } - late final ___swbufPtr = - _lookup)>>( - '__swbuf'); - late final ___swbuf = - ___swbufPtr.asFunction)>(); + late final _ggml_are_same_shapePtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, + ffi.Pointer)>>('ggml_are_same_shape'); + late final _ggml_are_same_shape = _ggml_are_same_shapePtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer)>(); - void flockfile( - ffi.Pointer arg0, + bool ggml_are_same_stride( + ffi.Pointer t0, + ffi.Pointer t1, ) { - return _flockfile( - arg0, + return _ggml_are_same_stride( + t0, + t1, ); } - late final _flockfilePtr = - _lookup)>>( - 'flockfile'); - late final _flockfile = - _flockfilePtr.asFunction)>(); + late final _ggml_are_same_stridePtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, + ffi.Pointer)>>('ggml_are_same_stride'); + late final _ggml_are_same_stride = _ggml_are_same_stridePtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer)>(); - int ftrylockfile( - ffi.Pointer arg0, + ffi.Pointer ggml_argmax( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ftrylockfile( - arg0, + return _ggml_argmax( + ctx, + a, ); } - late final _ftrylockfilePtr = - _lookup)>>( - 'ftrylockfile'); - late final _ftrylockfile = - _ftrylockfilePtr.asFunction)>(); + late final _ggml_argmaxPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_argmax'); + late final _ggml_argmax = _ggml_argmaxPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void funlockfile( - ffi.Pointer arg0, + ffi.Pointer ggml_argsort( + ffi.Pointer ctx, + ffi.Pointer a, + ggml_sort_order order, ) { - return _funlockfile( - arg0, + return _ggml_argsort( + ctx, + a, + order.value, ); } - late final _funlockfilePtr = - _lookup)>>( - 'funlockfile'); - late final _funlockfile = - _funlockfilePtr.asFunction)>(); + late final _ggml_argsortPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.UnsignedInt)>>('ggml_argsort'); + late final _ggml_argsort = _ggml_argsortPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - int getc_unlocked( - ffi.Pointer arg0, + ffi.Pointer ggml_argsort_top_k( + ffi.Pointer ctx, + ffi.Pointer a, + int k, ) { - return _getc_unlocked( - arg0, + return _ggml_argsort_top_k( + ctx, + a, + k, ); } - late final _getc_unlockedPtr = - _lookup)>>( - 'getc_unlocked'); - late final _getc_unlocked = - _getc_unlockedPtr.asFunction)>(); - - int getchar_unlocked() { - return _getchar_unlocked(); - } - - late final _getchar_unlockedPtr = - _lookup>('getchar_unlocked'); - late final _getchar_unlocked = - _getchar_unlockedPtr.asFunction(); + late final _ggml_argsort_top_kPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Int)>>('ggml_argsort_top_k'); + late final _ggml_argsort_top_k = _ggml_argsort_top_kPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - int putc_unlocked( - int arg0, - ffi.Pointer arg1, + ggml_backend_buffer_t ggml_backend_alloc_buffer( + ggml_backend_t backend, + int size, ) { - return _putc_unlocked( - arg0, - arg1, + return _ggml_backend_alloc_buffer( + backend, + size, ); } - late final _putc_unlockedPtr = - _lookup)>>( - 'putc_unlocked'); - late final _putc_unlocked = - _putc_unlockedPtr.asFunction)>(); + late final _ggml_backend_alloc_bufferPtr = _lookup< + ffi.NativeFunction< + ggml_backend_buffer_t Function( + ggml_backend_t, ffi.Size)>>('ggml_backend_alloc_buffer'); + late final _ggml_backend_alloc_buffer = _ggml_backend_alloc_bufferPtr + .asFunction(); - int putchar_unlocked( - int arg0, + ffi.Pointer ggml_backend_alloc_ctx_tensors( + ffi.Pointer ctx, + ggml_backend_t backend, ) { - return _putchar_unlocked( - arg0, + return _ggml_backend_alloc_ctx_tensors( + ctx, + backend, ); } - late final _putchar_unlockedPtr = - _lookup>( - 'putchar_unlocked'); - late final _putchar_unlocked = - _putchar_unlockedPtr.asFunction(); + late final _ggml_backend_alloc_ctx_tensorsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ggml_backend_t)>>('ggml_backend_alloc_ctx_tensors'); + late final _ggml_backend_alloc_ctx_tensors = + _ggml_backend_alloc_ctx_tensorsPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ggml_backend_t)>(); - int getw( - ffi.Pointer arg0, + ffi.Pointer ggml_backend_alloc_ctx_tensors_from_buft( + ffi.Pointer ctx, + ggml_backend_buffer_type_t buft, ) { - return _getw( - arg0, + return _ggml_backend_alloc_ctx_tensors_from_buft( + ctx, + buft, ); } - late final _getwPtr = - _lookup)>>('getw'); - late final _getw = _getwPtr.asFunction)>(); + late final _ggml_backend_alloc_ctx_tensors_from_buftPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ggml_backend_buffer_type_t)>>( + 'ggml_backend_alloc_ctx_tensors_from_buft'); + late final _ggml_backend_alloc_ctx_tensors_from_buft = + _ggml_backend_alloc_ctx_tensors_from_buftPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ggml_backend_buffer_type_t)>(); - int putw( - int arg0, - ffi.Pointer arg1, + int ggml_backend_alloc_ctx_tensors_from_buft_size( + ffi.Pointer ctx, + ggml_backend_buffer_type_t buft, ) { - return _putw( - arg0, - arg1, + return _ggml_backend_alloc_ctx_tensors_from_buft_size( + ctx, + buft, ); } - late final _putwPtr = - _lookup)>>( - 'putw'); - late final _putw = - _putwPtr.asFunction)>(); - - ffi.Pointer tempnam( - ffi.Pointer __dir, - ffi.Pointer __prefix, + late final _ggml_backend_alloc_ctx_tensors_from_buft_sizePtr = _lookup< + ffi.NativeFunction< + ffi.Size Function( + ffi.Pointer, ggml_backend_buffer_type_t)>>( + 'ggml_backend_alloc_ctx_tensors_from_buft_size'); + late final _ggml_backend_alloc_ctx_tensors_from_buft_size = + _ggml_backend_alloc_ctx_tensors_from_buft_sizePtr.asFunction< + int Function( + ffi.Pointer, ggml_backend_buffer_type_t)>(); + + void ggml_backend_buffer_clear( + ggml_backend_buffer_t buffer, + int value, ) { - return _tempnam( - __dir, - __prefix, + return _ggml_backend_buffer_clear( + buffer, + value, ); } - late final _tempnamPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>>('tempnam'); - late final _tempnam = _tempnamPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_backend_buffer_clearPtr = _lookup< + ffi + .NativeFunction>( + 'ggml_backend_buffer_clear'); + late final _ggml_backend_buffer_clear = _ggml_backend_buffer_clearPtr + .asFunction(); - int fseeko( - ffi.Pointer __stream, - int __offset, - int __whence, + void ggml_backend_buffer_free( + ggml_backend_buffer_t buffer, ) { - return _fseeko( - __stream, - __offset, - __whence, + return _ggml_backend_buffer_free( + buffer, ); } - late final _fseekoPtr = _lookup< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, off_t, ffi.Int)>>('fseeko'); - late final _fseeko = - _fseekoPtr.asFunction, int, int)>(); + late final _ggml_backend_buffer_freePtr = + _lookup>( + 'ggml_backend_buffer_free'); + late final _ggml_backend_buffer_free = _ggml_backend_buffer_freePtr + .asFunction(); - int ftello( - ffi.Pointer __stream, + int ggml_backend_buffer_get_alignment( + ggml_backend_buffer_t buffer, ) { - return _ftello( - __stream, + return _ggml_backend_buffer_get_alignment( + buffer, ); } - late final _ftelloPtr = - _lookup)>>('ftello'); - late final _ftello = _ftelloPtr.asFunction)>(); + late final _ggml_backend_buffer_get_alignmentPtr = + _lookup>( + 'ggml_backend_buffer_get_alignment'); + late final _ggml_backend_buffer_get_alignment = + _ggml_backend_buffer_get_alignmentPtr + .asFunction(); - int snprintf( - ffi.Pointer __str, - int __size, - ffi.Pointer __format, + int ggml_backend_buffer_get_alloc_size( + ggml_backend_buffer_t buffer, + ffi.Pointer tensor, ) { - return _snprintf( - __str, - __size, - __format, + return _ggml_backend_buffer_get_alloc_size( + buffer, + tensor, ); } - late final _snprintfPtr = _lookup< + late final _ggml_backend_buffer_get_alloc_sizePtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Size, - ffi.Pointer)>>('snprintf'); - late final _snprintf = _snprintfPtr.asFunction< - int Function(ffi.Pointer, int, ffi.Pointer)>(); + ffi.Size Function(ggml_backend_buffer_t, + ffi.Pointer)>>('ggml_backend_buffer_get_alloc_size'); + late final _ggml_backend_buffer_get_alloc_size = + _ggml_backend_buffer_get_alloc_sizePtr.asFunction< + int Function(ggml_backend_buffer_t, ffi.Pointer)>(); - int vfscanf( - ffi.Pointer __stream, - ffi.Pointer __format, - va_list arg2, + ffi.Pointer ggml_backend_buffer_get_base( + ggml_backend_buffer_t buffer, ) { - return _vfscanf( - __stream, - __format, - arg2, + return _ggml_backend_buffer_get_base( + buffer, ); } - late final _vfscanfPtr = _lookup< + late final _ggml_backend_buffer_get_basePtr = _lookup< ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer, va_list)>>('vfscanf'); - late final _vfscanf = _vfscanfPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, va_list)>(); + ffi.Pointer Function( + ggml_backend_buffer_t)>>('ggml_backend_buffer_get_base'); + late final _ggml_backend_buffer_get_base = _ggml_backend_buffer_get_basePtr + .asFunction Function(ggml_backend_buffer_t)>(); - int vscanf( - ffi.Pointer __format, - va_list arg1, + int ggml_backend_buffer_get_max_size( + ggml_backend_buffer_t buffer, ) { - return _vscanf( - __format, - arg1, + return _ggml_backend_buffer_get_max_size( + buffer, ); } - late final _vscanfPtr = _lookup< - ffi.NativeFunction, va_list)>>( - 'vscanf'); - late final _vscanf = - _vscanfPtr.asFunction, va_list)>(); + late final _ggml_backend_buffer_get_max_sizePtr = + _lookup>( + 'ggml_backend_buffer_get_max_size'); + late final _ggml_backend_buffer_get_max_size = + _ggml_backend_buffer_get_max_sizePtr + .asFunction(); - int vsnprintf( - ffi.Pointer __str, - int __size, - ffi.Pointer __format, - va_list arg3, + int ggml_backend_buffer_get_size( + ggml_backend_buffer_t buffer, ) { - return _vsnprintf( - __str, - __size, - __format, - arg3, + return _ggml_backend_buffer_get_size( + buffer, ); } - late final _vsnprintfPtr = _lookup< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Size, - ffi.Pointer, va_list)>>('vsnprintf'); - late final _vsnprintf = _vsnprintfPtr.asFunction< - int Function( - ffi.Pointer, int, ffi.Pointer, va_list)>(); + late final _ggml_backend_buffer_get_sizePtr = + _lookup>( + 'ggml_backend_buffer_get_size'); + late final _ggml_backend_buffer_get_size = _ggml_backend_buffer_get_sizePtr + .asFunction(); - int vsscanf( - ffi.Pointer __str, - ffi.Pointer __format, - va_list arg2, + ggml_backend_buffer_type_t ggml_backend_buffer_get_type( + ggml_backend_buffer_t buffer, ) { - return _vsscanf( - __str, - __format, - arg2, + return _ggml_backend_buffer_get_type( + buffer, ); } - late final _vsscanfPtr = _lookup< + late final _ggml_backend_buffer_get_typePtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, - va_list)>>('vsscanf'); - late final _vsscanf = _vsscanfPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, va_list)>(); + ggml_backend_buffer_type_t Function( + ggml_backend_buffer_t)>>('ggml_backend_buffer_get_type'); + late final _ggml_backend_buffer_get_type = _ggml_backend_buffer_get_typePtr + .asFunction(); - int dprintf( - int arg0, - ffi.Pointer arg1, + ggml_backend_buffer_usage ggml_backend_buffer_get_usage( + ggml_backend_buffer_t buffer, ) { - return _dprintf( - arg0, - arg1, - ); + return ggml_backend_buffer_usage.fromValue(_ggml_backend_buffer_get_usage( + buffer, + )); } - late final _dprintfPtr = _lookup< - ffi.NativeFunction)>>( - 'dprintf'); - late final _dprintf = - _dprintfPtr.asFunction)>(); + late final _ggml_backend_buffer_get_usagePtr = _lookup< + ffi.NativeFunction>( + 'ggml_backend_buffer_get_usage'); + late final _ggml_backend_buffer_get_usage = _ggml_backend_buffer_get_usagePtr + .asFunction(); - int vdprintf( - int arg0, - ffi.Pointer arg1, - va_list arg2, + ggml_status ggml_backend_buffer_init_tensor( + ggml_backend_buffer_t buffer, + ffi.Pointer tensor, ) { - return _vdprintf( - arg0, - arg1, - arg2, - ); + return ggml_status.fromValue(_ggml_backend_buffer_init_tensor( + buffer, + tensor, + )); } - late final _vdprintfPtr = _lookup< + late final _ggml_backend_buffer_init_tensorPtr = _lookup< ffi.NativeFunction< - ffi.Int Function( - ffi.Int, ffi.Pointer, va_list)>>('vdprintf'); - late final _vdprintf = _vdprintfPtr - .asFunction, va_list)>(); + ffi.Int Function(ggml_backend_buffer_t, + ffi.Pointer)>>('ggml_backend_buffer_init_tensor'); + late final _ggml_backend_buffer_init_tensor = + _ggml_backend_buffer_init_tensorPtr.asFunction< + int Function(ggml_backend_buffer_t, ffi.Pointer)>(); - int getdelim( - ffi.Pointer> __linep, - ffi.Pointer __linecapp, - int __delimiter, - ffi.Pointer __stream, + bool ggml_backend_buffer_is_host( + ggml_backend_buffer_t buffer, ) { - return _getdelim( - __linep, - __linecapp, - __delimiter, - __stream, + return _ggml_backend_buffer_is_host( + buffer, ); } - late final _getdelimPtr = _lookup< - ffi.NativeFunction< - ssize_t Function(ffi.Pointer>, - ffi.Pointer, ffi.Int, ffi.Pointer)>>('getdelim'); - late final _getdelim = _getdelimPtr.asFunction< - int Function(ffi.Pointer>, ffi.Pointer, - int, ffi.Pointer)>(); + late final _ggml_backend_buffer_is_hostPtr = + _lookup>( + 'ggml_backend_buffer_is_host'); + late final _ggml_backend_buffer_is_host = _ggml_backend_buffer_is_hostPtr + .asFunction(); - int getline( - ffi.Pointer> __linep, - ffi.Pointer __linecapp, - ffi.Pointer __stream, + ffi.Pointer ggml_backend_buffer_name( + ggml_backend_buffer_t buffer, ) { - return _getline( - __linep, - __linecapp, - __stream, + return _ggml_backend_buffer_name( + buffer, ); } - late final _getlinePtr = _lookup< + late final _ggml_backend_buffer_namePtr = _lookup< ffi.NativeFunction< - ssize_t Function(ffi.Pointer>, - ffi.Pointer, ffi.Pointer)>>('getline'); - late final _getline = _getlinePtr.asFunction< - int Function(ffi.Pointer>, ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function( + ggml_backend_buffer_t)>>('ggml_backend_buffer_name'); + late final _ggml_backend_buffer_name = _ggml_backend_buffer_namePtr + .asFunction Function(ggml_backend_buffer_t)>(); - ffi.Pointer fmemopen( - ffi.Pointer __buf, - int __size, - ffi.Pointer __mode, + void ggml_backend_buffer_reset( + ggml_backend_buffer_t buffer, ) { - return _fmemopen( - __buf, - __size, - __mode, + return _ggml_backend_buffer_reset( + buffer, ); } - late final _fmemopenPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, ffi.Size, - ffi.Pointer)>>('fmemopen'); - late final _fmemopen = _fmemopenPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int, ffi.Pointer)>(); + late final _ggml_backend_buffer_resetPtr = + _lookup>( + 'ggml_backend_buffer_reset'); + late final _ggml_backend_buffer_reset = _ggml_backend_buffer_resetPtr + .asFunction(); - ffi.Pointer open_memstream( - ffi.Pointer> __bufp, - ffi.Pointer __sizep, + void ggml_backend_buffer_set_usage( + ggml_backend_buffer_t buffer, + ggml_backend_buffer_usage usage, ) { - return _open_memstream( - __bufp, - __sizep, + return _ggml_backend_buffer_set_usage( + buffer, + usage.value, ); } - late final _open_memstreamPtr = _lookup< + late final _ggml_backend_buffer_set_usagePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer>, - ffi.Pointer)>>('open_memstream'); - late final _open_memstream = _open_memstreamPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer>, ffi.Pointer)>(); - - late final ffi.Pointer _sys_nerr = _lookup('sys_nerr'); - - int get sys_nerr => _sys_nerr.value; - - late final ffi.Pointer>> _sys_errlist = - _lookup>>('sys_errlist'); - - ffi.Pointer> get sys_errlist => _sys_errlist.value; - - set sys_errlist(ffi.Pointer> value) => - _sys_errlist.value = value; + ffi.Void Function(ggml_backend_buffer_t, + ffi.UnsignedInt)>>('ggml_backend_buffer_set_usage'); + late final _ggml_backend_buffer_set_usage = _ggml_backend_buffer_set_usagePtr + .asFunction(); - int asprintf( - ffi.Pointer> arg0, - ffi.Pointer arg1, + ggml_backend_buffer_t ggml_backend_buft_alloc_buffer( + ggml_backend_buffer_type_t buft, + int size, ) { - return _asprintf( - arg0, - arg1, + return _ggml_backend_buft_alloc_buffer( + buft, + size, ); } - late final _asprintfPtr = _lookup< + late final _ggml_backend_buft_alloc_bufferPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer>, - ffi.Pointer)>>('asprintf'); - late final _asprintf = _asprintfPtr.asFunction< - int Function( - ffi.Pointer>, ffi.Pointer)>(); + ggml_backend_buffer_t Function(ggml_backend_buffer_type_t, + ffi.Size)>>('ggml_backend_buft_alloc_buffer'); + late final _ggml_backend_buft_alloc_buffer = + _ggml_backend_buft_alloc_bufferPtr.asFunction< + ggml_backend_buffer_t Function(ggml_backend_buffer_type_t, int)>(); - ffi.Pointer ctermid_r( - ffi.Pointer arg0, + int ggml_backend_buft_get_alignment( + ggml_backend_buffer_type_t buft, ) { - return _ctermid_r( - arg0, + return _ggml_backend_buft_get_alignment( + buft, ); } - late final _ctermid_rPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer)>>('ctermid_r'); - late final _ctermid_r = _ctermid_rPtr - .asFunction Function(ffi.Pointer)>(); + late final _ggml_backend_buft_get_alignmentPtr = _lookup< + ffi.NativeFunction>( + 'ggml_backend_buft_get_alignment'); + late final _ggml_backend_buft_get_alignment = + _ggml_backend_buft_get_alignmentPtr + .asFunction(); - ffi.Pointer fgetln( - ffi.Pointer arg0, - ffi.Pointer __len, + int ggml_backend_buft_get_alloc_size( + ggml_backend_buffer_type_t buft, + ffi.Pointer tensor, ) { - return _fgetln( - arg0, - __len, + return _ggml_backend_buft_get_alloc_size( + buft, + tensor, ); } - late final _fgetlnPtr = _lookup< + late final _ggml_backend_buft_get_alloc_sizePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>>('fgetln'); - late final _fgetln = _fgetlnPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Size Function(ggml_backend_buffer_type_t, + ffi.Pointer)>>('ggml_backend_buft_get_alloc_size'); + late final _ggml_backend_buft_get_alloc_size = + _ggml_backend_buft_get_alloc_sizePtr.asFunction< + int Function(ggml_backend_buffer_type_t, ffi.Pointer)>(); - ffi.Pointer fmtcheck( - ffi.Pointer arg0, - ffi.Pointer arg1, + ggml_backend_dev_t ggml_backend_buft_get_device( + ggml_backend_buffer_type_t buft, ) { - return _fmtcheck( - arg0, - arg1, + return _ggml_backend_buft_get_device( + buft, ); } - late final _fmtcheckPtr = _lookup< + late final _ggml_backend_buft_get_devicePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>>('fmtcheck'); - late final _fmtcheck = _fmtcheckPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); - - int fpurge( - ffi.Pointer arg0, - ) { - return _fpurge( - arg0, - ); - } - - late final _fpurgePtr = - _lookup)>>( - 'fpurge'); - late final _fpurge = _fpurgePtr.asFunction)>(); + ggml_backend_dev_t Function( + ggml_backend_buffer_type_t)>>('ggml_backend_buft_get_device'); + late final _ggml_backend_buft_get_device = _ggml_backend_buft_get_devicePtr + .asFunction(); - void setbuffer( - ffi.Pointer arg0, - ffi.Pointer arg1, - int __size, + int ggml_backend_buft_get_max_size( + ggml_backend_buffer_type_t buft, ) { - return _setbuffer( - arg0, - arg1, - __size, + return _ggml_backend_buft_get_max_size( + buft, ); } - late final _setbufferPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, ffi.Pointer, ffi.Int)>>('setbuffer'); - late final _setbuffer = _setbufferPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_backend_buft_get_max_sizePtr = _lookup< + ffi.NativeFunction>( + 'ggml_backend_buft_get_max_size'); + late final _ggml_backend_buft_get_max_size = + _ggml_backend_buft_get_max_sizePtr + .asFunction(); - int setlinebuf( - ffi.Pointer arg0, + bool ggml_backend_buft_is_host( + ggml_backend_buffer_type_t buft, ) { - return _setlinebuf( - arg0, + return _ggml_backend_buft_is_host( + buft, ); } - late final _setlinebufPtr = - _lookup)>>( - 'setlinebuf'); - late final _setlinebuf = - _setlinebufPtr.asFunction)>(); + late final _ggml_backend_buft_is_hostPtr = _lookup< + ffi.NativeFunction>( + 'ggml_backend_buft_is_host'); + late final _ggml_backend_buft_is_host = _ggml_backend_buft_is_hostPtr + .asFunction(); - int vasprintf( - ffi.Pointer> arg0, - ffi.Pointer arg1, - va_list arg2, + ffi.Pointer ggml_backend_buft_name( + ggml_backend_buffer_type_t buft, ) { - return _vasprintf( - arg0, - arg1, - arg2, + return _ggml_backend_buft_name( + buft, ); } - late final _vasprintfPtr = _lookup< + late final _ggml_backend_buft_namePtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer>, - ffi.Pointer, va_list)>>('vasprintf'); - late final _vasprintf = _vasprintfPtr.asFunction< - int Function(ffi.Pointer>, ffi.Pointer, - va_list)>(); + ffi.Pointer Function( + ggml_backend_buffer_type_t)>>('ggml_backend_buft_name'); + late final _ggml_backend_buft_name = _ggml_backend_buft_namePtr + .asFunction Function(ggml_backend_buffer_type_t)>(); - ffi.Pointer funopen( - ffi.Pointer arg0, - ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer, ffi.Int)>> - arg1, - ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer, ffi.Int)>> - arg2, - ffi.Pointer< - ffi.NativeFunction< - fpos_t Function(ffi.Pointer, fpos_t, ffi.Int)>> - arg3, - ffi.Pointer)>> - arg4, + bool ggml_backend_compare_graph_backend( + ggml_backend_t backend1, + ggml_backend_t backend2, + ffi.Pointer graph, + ggml_backend_eval_callback callback, + ffi.Pointer user_data, + ffi.Pointer> test_nodes, + int num_test_nodes, ) { - return _funopen( - arg0, - arg1, - arg2, - arg3, - arg4, + return _ggml_backend_compare_graph_backend( + backend1, + backend2, + graph, + callback, + user_data, + test_nodes, + num_test_nodes, ); } - late final _funopenPtr = _lookup< + late final _ggml_backend_compare_graph_backendPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( + ffi.Bool Function( + ggml_backend_t, + ggml_backend_t, + ffi.Pointer, + ggml_backend_eval_callback, ffi.Pointer, - ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, - ffi.Pointer, ffi.Int)>>, - ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, - ffi.Pointer, ffi.Int)>>, - ffi.Pointer< - ffi.NativeFunction< - fpos_t Function(ffi.Pointer, fpos_t, ffi.Int)>>, - ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer)>>)>>('funopen'); - late final _funopen = _funopenPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer, ffi.Int)>>, - ffi.Pointer< - ffi.NativeFunction< - ffi.Int Function( - ffi.Pointer, ffi.Pointer, ffi.Int)>>, - ffi.Pointer< - ffi.NativeFunction< - fpos_t Function(ffi.Pointer, fpos_t, ffi.Int)>>, - ffi.Pointer< - ffi.NativeFunction)>>)>(); + ffi.Pointer>, + ffi.Size)>>('ggml_backend_compare_graph_backend'); + late final _ggml_backend_compare_graph_backend = + _ggml_backend_compare_graph_backendPtr.asFunction< + bool Function( + ggml_backend_t, + ggml_backend_t, + ffi.Pointer, + ggml_backend_eval_callback, + ffi.Pointer, + ffi.Pointer>, + int)>(); - int __snprintf_chk( - ffi.Pointer arg0, - int __maxlen, - int arg2, - int arg3, - ffi.Pointer arg4, + ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr( + ffi.Pointer ptr, + int size, ) { - return ___snprintf_chk( - arg0, - __maxlen, - arg2, - arg3, - arg4, + return _ggml_backend_cpu_buffer_from_ptr( + ptr, + size, ); } - late final ___snprintf_chkPtr = _lookup< + late final _ggml_backend_cpu_buffer_from_ptrPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Size, ffi.Int, ffi.Size, - ffi.Pointer)>>('__snprintf_chk'); - late final ___snprintf_chk = ___snprintf_chkPtr.asFunction< - int Function( - ffi.Pointer, int, int, int, ffi.Pointer)>(); + ggml_backend_buffer_t Function(ffi.Pointer, + ffi.Size)>>('ggml_backend_cpu_buffer_from_ptr'); + late final _ggml_backend_cpu_buffer_from_ptr = + _ggml_backend_cpu_buffer_from_ptrPtr.asFunction< + ggml_backend_buffer_t Function(ffi.Pointer, int)>(); - int __vsnprintf_chk( - ffi.Pointer arg0, - int __maxlen, - int arg2, - int arg3, - ffi.Pointer arg4, - va_list arg5, - ) { - return ___vsnprintf_chk( - arg0, - __maxlen, - arg2, - arg3, - arg4, - arg5, - ); + ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type() { + return _ggml_backend_cpu_buffer_type(); } - late final ___vsnprintf_chkPtr = _lookup< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Size, ffi.Int, ffi.Size, - ffi.Pointer, va_list)>>('__vsnprintf_chk'); - late final ___vsnprintf_chk = ___vsnprintf_chkPtr.asFunction< - int Function(ffi.Pointer, int, int, int, ffi.Pointer, - va_list)>(); + late final _ggml_backend_cpu_buffer_typePtr = + _lookup>( + 'ggml_backend_cpu_buffer_type'); + late final _ggml_backend_cpu_buffer_type = _ggml_backend_cpu_buffer_typePtr + .asFunction(); - int __sprintf_chk( - ffi.Pointer arg0, - int arg1, - int arg2, - ffi.Pointer arg3, - ) { - return ___sprintf_chk( - arg0, - arg1, - arg2, - arg3, - ); + ggml_backend_t ggml_backend_cpu_init() { + return _ggml_backend_cpu_init(); } - late final ___sprintf_chkPtr = _lookup< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Int, ffi.Size, - ffi.Pointer)>>('__sprintf_chk'); - late final ___sprintf_chk = ___sprintf_chkPtr.asFunction< - int Function(ffi.Pointer, int, int, ffi.Pointer)>(); + late final _ggml_backend_cpu_initPtr = + _lookup>( + 'ggml_backend_cpu_init'); + late final _ggml_backend_cpu_init = + _ggml_backend_cpu_initPtr.asFunction(); - int __vsprintf_chk( - ffi.Pointer arg0, - int arg1, - int arg2, - ffi.Pointer arg3, - va_list arg4, + ggml_backend_reg_t ggml_backend_cpu_reg() { + return _ggml_backend_cpu_reg(); + } + + late final _ggml_backend_cpu_regPtr = + _lookup>( + 'ggml_backend_cpu_reg'); + late final _ggml_backend_cpu_reg = + _ggml_backend_cpu_regPtr.asFunction(); + + void ggml_backend_cpu_set_abort_callback( + ggml_backend_t backend_cpu, + ggml_abort_callback abort_callback, + ffi.Pointer abort_callback_data, ) { - return ___vsprintf_chk( - arg0, - arg1, - arg2, - arg3, - arg4, + return _ggml_backend_cpu_set_abort_callback( + backend_cpu, + abort_callback, + abort_callback_data, ); } - late final ___vsprintf_chkPtr = _lookup< + late final _ggml_backend_cpu_set_abort_callbackPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Int, ffi.Size, - ffi.Pointer, va_list)>>('__vsprintf_chk'); - late final ___vsprintf_chk = ___vsprintf_chkPtr.asFunction< - int Function( - ffi.Pointer, int, int, ffi.Pointer, va_list)>(); + ffi.Void Function(ggml_backend_t, ggml_abort_callback, + ffi.Pointer)>>('ggml_backend_cpu_set_abort_callback'); + late final _ggml_backend_cpu_set_abort_callback = + _ggml_backend_cpu_set_abort_callbackPtr.asFunction< + void Function( + ggml_backend_t, ggml_abort_callback, ffi.Pointer)>(); - ggml_abort_callback_t ggml_set_abort_callback( - ggml_abort_callback_t callback, + void ggml_backend_cpu_set_n_threads( + ggml_backend_t backend_cpu, + int n_threads, ) { - return _ggml_set_abort_callback( - callback, + return _ggml_backend_cpu_set_n_threads( + backend_cpu, + n_threads, ); } - late final _ggml_set_abort_callbackPtr = _lookup< - ffi.NativeFunction< - ggml_abort_callback_t Function( - ggml_abort_callback_t)>>('ggml_set_abort_callback'); - late final _ggml_set_abort_callback = _ggml_set_abort_callbackPtr - .asFunction(); + late final _ggml_backend_cpu_set_n_threadsPtr = + _lookup>( + 'ggml_backend_cpu_set_n_threads'); + late final _ggml_backend_cpu_set_n_threads = + _ggml_backend_cpu_set_n_threadsPtr + .asFunction(); - void ggml_abort( - ffi.Pointer file, - int line, - ffi.Pointer fmt, + void ggml_backend_cpu_set_threadpool( + ggml_backend_t backend_cpu, + ggml_threadpool_t threadpool, ) { - return _ggml_abort( - file, - line, - fmt, + return _ggml_backend_cpu_set_threadpool( + backend_cpu, + threadpool, ); } - late final _ggml_abortPtr = _lookup< + late final _ggml_backend_cpu_set_threadpoolPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Int, - ffi.Pointer)>>('ggml_abort'); - late final _ggml_abort = _ggml_abortPtr.asFunction< - void Function(ffi.Pointer, int, ffi.Pointer)>(); + ffi.Void Function(ggml_backend_t, + ggml_threadpool_t)>>('ggml_backend_cpu_set_threadpool'); + late final _ggml_backend_cpu_set_threadpool = + _ggml_backend_cpu_set_threadpoolPtr + .asFunction(); - ffi.Pointer ggml_status_to_string( - ggml_status status, + void ggml_backend_cpu_set_use_ref( + ggml_backend_t backend_cpu, + bool use_ref, ) { - return _ggml_status_to_string( - status.value, + return _ggml_backend_cpu_set_use_ref( + backend_cpu, + use_ref, ); } - late final _ggml_status_to_stringPtr = - _lookup Function(ffi.Int)>>( - 'ggml_status_to_string'); - late final _ggml_status_to_string = _ggml_status_to_stringPtr - .asFunction Function(int)>(); + late final _ggml_backend_cpu_set_use_refPtr = + _lookup>( + 'ggml_backend_cpu_set_use_ref'); + late final _ggml_backend_cpu_set_use_ref = _ggml_backend_cpu_set_use_refPtr + .asFunction(); - double ggml_fp16_to_fp32( - int arg0, + ggml_backend_reg_t ggml_backend_dev_backend_reg( + ggml_backend_dev_t device, ) { - return _ggml_fp16_to_fp32( - arg0, + return _ggml_backend_dev_backend_reg( + device, ); } - late final _ggml_fp16_to_fp32Ptr = - _lookup>( - 'ggml_fp16_to_fp32'); - late final _ggml_fp16_to_fp32 = - _ggml_fp16_to_fp32Ptr.asFunction(); + late final _ggml_backend_dev_backend_regPtr = _lookup< + ffi.NativeFunction>( + 'ggml_backend_dev_backend_reg'); + late final _ggml_backend_dev_backend_reg = _ggml_backend_dev_backend_regPtr + .asFunction(); - int ggml_fp32_to_fp16( - double arg0, + ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr( + ggml_backend_dev_t device, + ffi.Pointer ptr, + int size, + int max_tensor_size, ) { - return _ggml_fp32_to_fp16( - arg0, + return _ggml_backend_dev_buffer_from_host_ptr( + device, + ptr, + size, + max_tensor_size, ); } - late final _ggml_fp32_to_fp16Ptr = - _lookup>( - 'ggml_fp32_to_fp16'); - late final _ggml_fp32_to_fp16 = - _ggml_fp32_to_fp16Ptr.asFunction(); + late final _ggml_backend_dev_buffer_from_host_ptrPtr = _lookup< + ffi.NativeFunction< + ggml_backend_buffer_t Function( + ggml_backend_dev_t, + ffi.Pointer, + ffi.Size, + ffi.Size)>>('ggml_backend_dev_buffer_from_host_ptr'); + late final _ggml_backend_dev_buffer_from_host_ptr = + _ggml_backend_dev_buffer_from_host_ptrPtr.asFunction< + ggml_backend_buffer_t Function( + ggml_backend_dev_t, ffi.Pointer, int, int)>(); - void ggml_fp16_to_fp32_row( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + ggml_backend_buffer_type_t ggml_backend_dev_buffer_type( + ggml_backend_dev_t device, ) { - return _ggml_fp16_to_fp32_row( - arg0, - arg1, - arg2, + return _ggml_backend_dev_buffer_type( + device, ); } - late final _ggml_fp16_to_fp32_rowPtr = _lookup< + late final _ggml_backend_dev_buffer_typePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_fp16_to_fp32_row'); - late final _ggml_fp16_to_fp32_row = _ggml_fp16_to_fp32_rowPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ggml_backend_buffer_type_t Function( + ggml_backend_dev_t)>>('ggml_backend_dev_buffer_type'); + late final _ggml_backend_dev_buffer_type = _ggml_backend_dev_buffer_typePtr + .asFunction(); - void ggml_fp32_to_fp16_row( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + ggml_backend_dev_t ggml_backend_dev_by_name( + ffi.Pointer name, ) { - return _ggml_fp32_to_fp16_row( - arg0, - arg1, - arg2, + return _ggml_backend_dev_by_name( + name, ); } - late final _ggml_fp32_to_fp16_rowPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_fp32_to_fp16_row'); - late final _ggml_fp32_to_fp16_row = _ggml_fp32_to_fp16_rowPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_backend_dev_by_namePtr = _lookup< + ffi + .NativeFunction)>>( + 'ggml_backend_dev_by_name'); + late final _ggml_backend_dev_by_name = _ggml_backend_dev_by_namePtr + .asFunction)>(); - ggml_bf16_t ggml_fp32_to_bf16( - double arg0, + ggml_backend_dev_t ggml_backend_dev_by_type( + ggml_backend_dev_type type, ) { - return _ggml_fp32_to_bf16( - arg0, + return _ggml_backend_dev_by_type( + type.value, ); } - late final _ggml_fp32_to_bf16Ptr = - _lookup>( - 'ggml_fp32_to_bf16'); - late final _ggml_fp32_to_bf16 = - _ggml_fp32_to_bf16Ptr.asFunction(); + late final _ggml_backend_dev_by_typePtr = + _lookup>( + 'ggml_backend_dev_by_type'); + late final _ggml_backend_dev_by_type = _ggml_backend_dev_by_typePtr + .asFunction(); - double ggml_bf16_to_fp32( - ggml_bf16_t arg0, + int ggml_backend_dev_count() { + return _ggml_backend_dev_count(); + } + + late final _ggml_backend_dev_countPtr = + _lookup>( + 'ggml_backend_dev_count'); + late final _ggml_backend_dev_count = + _ggml_backend_dev_countPtr.asFunction(); + + ffi.Pointer ggml_backend_dev_description( + ggml_backend_dev_t device, ) { - return _ggml_bf16_to_fp32( - arg0, + return _ggml_backend_dev_description( + device, ); } - late final _ggml_bf16_to_fp32Ptr = - _lookup>( - 'ggml_bf16_to_fp32'); - late final _ggml_bf16_to_fp32 = - _ggml_bf16_to_fp32Ptr.asFunction(); + late final _ggml_backend_dev_descriptionPtr = _lookup< + ffi + .NativeFunction Function(ggml_backend_dev_t)>>( + 'ggml_backend_dev_description'); + late final _ggml_backend_dev_description = _ggml_backend_dev_descriptionPtr + .asFunction Function(ggml_backend_dev_t)>(); - void ggml_bf16_to_fp32_row( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + ggml_backend_dev_t ggml_backend_dev_get( + int index, ) { - return _ggml_bf16_to_fp32_row( - arg0, - arg1, - arg2, + return _ggml_backend_dev_get( + index, ); } - late final _ggml_bf16_to_fp32_rowPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_bf16_to_fp32_row'); - late final _ggml_bf16_to_fp32_row = _ggml_bf16_to_fp32_rowPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_backend_dev_getPtr = + _lookup>( + 'ggml_backend_dev_get'); + late final _ggml_backend_dev_get = + _ggml_backend_dev_getPtr.asFunction(); - void ggml_fp32_to_bf16_row_ref( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + void ggml_backend_dev_get_props( + ggml_backend_dev_t device, + ffi.Pointer props, ) { - return _ggml_fp32_to_bf16_row_ref( - arg0, - arg1, - arg2, + return _ggml_backend_dev_get_props( + device, + props, ); } - late final _ggml_fp32_to_bf16_row_refPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_fp32_to_bf16_row_ref'); - late final _ggml_fp32_to_bf16_row_ref = - _ggml_fp32_to_bf16_row_refPtr.asFunction< + late final _ggml_backend_dev_get_propsPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ggml_backend_dev_t, ffi.Pointer)>>( + 'ggml_backend_dev_get_props'); + late final _ggml_backend_dev_get_props = + _ggml_backend_dev_get_propsPtr.asFunction< void Function( - ffi.Pointer, ffi.Pointer, int)>(); + ggml_backend_dev_t, ffi.Pointer)>(); - void ggml_fp32_to_bf16_row( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type( + ggml_backend_dev_t device, ) { - return _ggml_fp32_to_bf16_row( - arg0, - arg1, - arg2, + return _ggml_backend_dev_host_buffer_type( + device, ); } - late final _ggml_fp32_to_bf16_rowPtr = _lookup< + late final _ggml_backend_dev_host_buffer_typePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_fp32_to_bf16_row'); - late final _ggml_fp32_to_bf16_row = _ggml_fp32_to_bf16_rowPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); - - late final ffi.Pointer _GGML_TENSOR_SIZE = - _lookup('GGML_TENSOR_SIZE'); - - int get GGML_TENSOR_SIZE => _GGML_TENSOR_SIZE.value; + ggml_backend_buffer_type_t Function( + ggml_backend_dev_t)>>('ggml_backend_dev_host_buffer_type'); + late final _ggml_backend_dev_host_buffer_type = + _ggml_backend_dev_host_buffer_typePtr.asFunction< + ggml_backend_buffer_type_t Function(ggml_backend_dev_t)>(); - bool ggml_guid_matches( - ggml_guid_t guid_a, - ggml_guid_t guid_b, + ggml_backend_t ggml_backend_dev_init( + ggml_backend_dev_t device, + ffi.Pointer params, ) { - return _ggml_guid_matches( - guid_a, - guid_b, + return _ggml_backend_dev_init( + device, + params, ); } - late final _ggml_guid_matchesPtr = - _lookup>( - 'ggml_guid_matches'); - late final _ggml_guid_matches = _ggml_guid_matchesPtr - .asFunction(); - - ffi.Pointer ggml_version() { - return _ggml_version(); - } - - late final _ggml_versionPtr = - _lookup Function()>>( - 'ggml_version'); - late final _ggml_version = - _ggml_versionPtr.asFunction Function()>(); + late final _ggml_backend_dev_initPtr = _lookup< + ffi.NativeFunction< + ggml_backend_t Function(ggml_backend_dev_t, + ffi.Pointer)>>('ggml_backend_dev_init'); + late final _ggml_backend_dev_init = _ggml_backend_dev_initPtr.asFunction< + ggml_backend_t Function(ggml_backend_dev_t, ffi.Pointer)>(); - ffi.Pointer ggml_commit() { - return _ggml_commit(); + void ggml_backend_dev_memory( + ggml_backend_dev_t device, + ffi.Pointer free, + ffi.Pointer total, + ) { + return _ggml_backend_dev_memory( + device, + free, + total, + ); } - late final _ggml_commitPtr = - _lookup Function()>>( - 'ggml_commit'); - late final _ggml_commit = - _ggml_commitPtr.asFunction Function()>(); + late final _ggml_backend_dev_memoryPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ggml_backend_dev_t, ffi.Pointer, + ffi.Pointer)>>('ggml_backend_dev_memory'); + late final _ggml_backend_dev_memory = _ggml_backend_dev_memoryPtr.asFunction< + void Function( + ggml_backend_dev_t, ffi.Pointer, ffi.Pointer)>(); - void ggml_time_init() { - return _ggml_time_init(); + ffi.Pointer ggml_backend_dev_name( + ggml_backend_dev_t device, + ) { + return _ggml_backend_dev_name( + device, + ); } - late final _ggml_time_initPtr = - _lookup>('ggml_time_init'); - late final _ggml_time_init = _ggml_time_initPtr.asFunction(); + late final _ggml_backend_dev_namePtr = _lookup< + ffi + .NativeFunction Function(ggml_backend_dev_t)>>( + 'ggml_backend_dev_name'); + late final _ggml_backend_dev_name = _ggml_backend_dev_namePtr + .asFunction Function(ggml_backend_dev_t)>(); - int ggml_time_ms() { - return _ggml_time_ms(); + bool ggml_backend_dev_offload_op( + ggml_backend_dev_t device, + ffi.Pointer op, + ) { + return _ggml_backend_dev_offload_op( + device, + op, + ); } - late final _ggml_time_msPtr = - _lookup>('ggml_time_ms'); - late final _ggml_time_ms = _ggml_time_msPtr.asFunction(); + late final _ggml_backend_dev_offload_opPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ggml_backend_dev_t, + ffi.Pointer)>>('ggml_backend_dev_offload_op'); + late final _ggml_backend_dev_offload_op = + _ggml_backend_dev_offload_opPtr.asFunction< + bool Function(ggml_backend_dev_t, ffi.Pointer)>(); - int ggml_time_us() { - return _ggml_time_us(); + bool ggml_backend_dev_supports_buft( + ggml_backend_dev_t device, + ggml_backend_buffer_type_t buft, + ) { + return _ggml_backend_dev_supports_buft( + device, + buft, + ); } - late final _ggml_time_usPtr = - _lookup>('ggml_time_us'); - late final _ggml_time_us = _ggml_time_usPtr.asFunction(); + late final _ggml_backend_dev_supports_buftPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ggml_backend_dev_t, + ggml_backend_buffer_type_t)>>('ggml_backend_dev_supports_buft'); + late final _ggml_backend_dev_supports_buft = + _ggml_backend_dev_supports_buftPtr.asFunction< + bool Function(ggml_backend_dev_t, ggml_backend_buffer_type_t)>(); - int ggml_cycles() { - return _ggml_cycles(); + bool ggml_backend_dev_supports_op( + ggml_backend_dev_t device, + ffi.Pointer op, + ) { + return _ggml_backend_dev_supports_op( + device, + op, + ); } - late final _ggml_cyclesPtr = - _lookup>('ggml_cycles'); - late final _ggml_cycles = _ggml_cyclesPtr.asFunction(); + late final _ggml_backend_dev_supports_opPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ggml_backend_dev_t, + ffi.Pointer)>>('ggml_backend_dev_supports_op'); + late final _ggml_backend_dev_supports_op = + _ggml_backend_dev_supports_opPtr.asFunction< + bool Function(ggml_backend_dev_t, ffi.Pointer)>(); - int ggml_cycles_per_ms() { - return _ggml_cycles_per_ms(); + ggml_backend_dev_type ggml_backend_dev_type$1( + ggml_backend_dev_t device, + ) { + return ggml_backend_dev_type.fromValue(_ggml_backend_dev_type( + device, + )); } - late final _ggml_cycles_per_msPtr = - _lookup>('ggml_cycles_per_ms'); - late final _ggml_cycles_per_ms = - _ggml_cycles_per_msPtr.asFunction(); + late final _ggml_backend_dev_type$1Ptr = + _lookup>( + 'ggml_backend_dev_type'); + late final _ggml_backend_dev_type = _ggml_backend_dev_type$1Ptr + .asFunction(); - ffi.Pointer ggml_fopen( - ffi.Pointer fname, - ffi.Pointer mode, + void ggml_backend_device_register( + ggml_backend_dev_t device, ) { - return _ggml_fopen( - fname, - mode, + return _ggml_backend_device_register( + device, ); } - late final _ggml_fopenPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>>('ggml_fopen'); - late final _ggml_fopen = _ggml_fopenPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_backend_device_registerPtr = + _lookup>( + 'ggml_backend_device_register'); + late final _ggml_backend_device_register = _ggml_backend_device_registerPtr + .asFunction(); - void ggml_print_object( - ffi.Pointer obj, + void ggml_backend_event_free( + ggml_backend_event_t event, ) { - return _ggml_print_object( - obj, + return _ggml_backend_event_free( + event, ); } - late final _ggml_print_objectPtr = - _lookup)>>( - 'ggml_print_object'); - late final _ggml_print_object = _ggml_print_objectPtr - .asFunction)>(); + late final _ggml_backend_event_freePtr = + _lookup>( + 'ggml_backend_event_free'); + late final _ggml_backend_event_free = _ggml_backend_event_freePtr + .asFunction(); - void ggml_print_objects( - ffi.Pointer ctx, + ggml_backend_event_t ggml_backend_event_new( + ggml_backend_dev_t device, ) { - return _ggml_print_objects( - ctx, + return _ggml_backend_event_new( + device, ); } - late final _ggml_print_objectsPtr = - _lookup)>>( - 'ggml_print_objects'); - late final _ggml_print_objects = _ggml_print_objectsPtr - .asFunction)>(); + late final _ggml_backend_event_newPtr = _lookup< + ffi + .NativeFunction>( + 'ggml_backend_event_new'); + late final _ggml_backend_event_new = _ggml_backend_event_newPtr + .asFunction(); - int ggml_nelements( - ffi.Pointer tensor, + void ggml_backend_event_record( + ggml_backend_event_t event, + ggml_backend_t backend, ) { - return _ggml_nelements( - tensor, + return _ggml_backend_event_record( + event, + backend, ); } - late final _ggml_nelementsPtr = - _lookup)>>( - 'ggml_nelements'); - late final _ggml_nelements = - _ggml_nelementsPtr.asFunction)>(); + late final _ggml_backend_event_recordPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ggml_backend_event_t, + ggml_backend_t)>>('ggml_backend_event_record'); + late final _ggml_backend_event_record = _ggml_backend_event_recordPtr + .asFunction(); - int ggml_nrows( - ffi.Pointer tensor, + void ggml_backend_event_synchronize( + ggml_backend_event_t event, ) { - return _ggml_nrows( - tensor, + return _ggml_backend_event_synchronize( + event, ); } - late final _ggml_nrowsPtr = - _lookup)>>( - 'ggml_nrows'); - late final _ggml_nrows = - _ggml_nrowsPtr.asFunction)>(); + late final _ggml_backend_event_synchronizePtr = + _lookup>( + 'ggml_backend_event_synchronize'); + late final _ggml_backend_event_synchronize = + _ggml_backend_event_synchronizePtr + .asFunction(); - int ggml_nbytes( - ffi.Pointer tensor, + void ggml_backend_event_wait( + ggml_backend_t backend, + ggml_backend_event_t event, ) { - return _ggml_nbytes( - tensor, + return _ggml_backend_event_wait( + backend, + event, ); } - late final _ggml_nbytesPtr = - _lookup)>>( - 'ggml_nbytes'); - late final _ggml_nbytes = - _ggml_nbytesPtr.asFunction)>(); - - int ggml_nbytes_pad( - ffi.Pointer tensor, - ) { - return _ggml_nbytes_pad( - tensor, - ); - } - - late final _ggml_nbytes_padPtr = - _lookup)>>( - 'ggml_nbytes_pad'); - late final _ggml_nbytes_pad = - _ggml_nbytes_padPtr.asFunction)>(); + late final _ggml_backend_event_waitPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ggml_backend_t, + ggml_backend_event_t)>>('ggml_backend_event_wait'); + late final _ggml_backend_event_wait = _ggml_backend_event_waitPtr + .asFunction(); - int ggml_blck_size( - ggml_type type, + void ggml_backend_free( + ggml_backend_t backend, ) { - return _ggml_blck_size( - type.value, + return _ggml_backend_free( + backend, ); } - late final _ggml_blck_sizePtr = - _lookup>( - 'ggml_blck_size'); - late final _ggml_blck_size = - _ggml_blck_sizePtr.asFunction(); + late final _ggml_backend_freePtr = + _lookup>( + 'ggml_backend_free'); + late final _ggml_backend_free = + _ggml_backend_freePtr.asFunction(); - int ggml_type_size( - ggml_type type, + int ggml_backend_get_alignment( + ggml_backend_t backend, ) { - return _ggml_type_size( - type.value, + return _ggml_backend_get_alignment( + backend, ); } - late final _ggml_type_sizePtr = - _lookup>( - 'ggml_type_size'); - late final _ggml_type_size = - _ggml_type_sizePtr.asFunction(); + late final _ggml_backend_get_alignmentPtr = + _lookup>( + 'ggml_backend_get_alignment'); + late final _ggml_backend_get_alignment = + _ggml_backend_get_alignmentPtr.asFunction(); - int ggml_row_size( - ggml_type type, - int ne, + ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type( + ggml_backend_t backend, ) { - return _ggml_row_size( - type.value, - ne, + return _ggml_backend_get_default_buffer_type( + backend, ); } - late final _ggml_row_sizePtr = _lookup< - ffi.NativeFunction>( - 'ggml_row_size'); - late final _ggml_row_size = - _ggml_row_sizePtr.asFunction(); + late final _ggml_backend_get_default_buffer_typePtr = _lookup< + ffi + .NativeFunction>( + 'ggml_backend_get_default_buffer_type'); + late final _ggml_backend_get_default_buffer_type = + _ggml_backend_get_default_buffer_typePtr + .asFunction(); - double ggml_type_sizef( - ggml_type type, + ggml_backend_dev_t ggml_backend_get_device( + ggml_backend_t backend, ) { - return _ggml_type_sizef( - type.value, + return _ggml_backend_get_device( + backend, ); } - late final _ggml_type_sizefPtr = - _lookup>( - 'ggml_type_sizef'); - late final _ggml_type_sizef = - _ggml_type_sizefPtr.asFunction(); + late final _ggml_backend_get_devicePtr = + _lookup>( + 'ggml_backend_get_device'); + late final _ggml_backend_get_device = _ggml_backend_get_devicePtr + .asFunction(); - ffi.Pointer ggml_type_name( - ggml_type type, + int ggml_backend_get_max_size( + ggml_backend_t backend, ) { - return _ggml_type_name( - type.value, + return _ggml_backend_get_max_size( + backend, ); } - late final _ggml_type_namePtr = _lookup< - ffi.NativeFunction Function(ffi.UnsignedInt)>>( - 'ggml_type_name'); - late final _ggml_type_name = - _ggml_type_namePtr.asFunction Function(int)>(); + late final _ggml_backend_get_max_sizePtr = + _lookup>( + 'ggml_backend_get_max_size'); + late final _ggml_backend_get_max_size = + _ggml_backend_get_max_sizePtr.asFunction(); - ffi.Pointer ggml_op_name( - ggml_op op, + ggml_status ggml_backend_graph_compute( + ggml_backend_t backend, + ffi.Pointer cgraph, ) { - return _ggml_op_name( - op.value, - ); + return ggml_status.fromValue(_ggml_backend_graph_compute( + backend, + cgraph, + )); } - late final _ggml_op_namePtr = _lookup< - ffi.NativeFunction Function(ffi.UnsignedInt)>>( - 'ggml_op_name'); - late final _ggml_op_name = - _ggml_op_namePtr.asFunction Function(int)>(); + late final _ggml_backend_graph_computePtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ggml_backend_t, + ffi.Pointer)>>('ggml_backend_graph_compute'); + late final _ggml_backend_graph_compute = _ggml_backend_graph_computePtr + .asFunction)>(); - ffi.Pointer ggml_op_symbol( - ggml_op op, + ggml_status ggml_backend_graph_compute_async( + ggml_backend_t backend, + ffi.Pointer cgraph, ) { - return _ggml_op_symbol( - op.value, - ); + return ggml_status.fromValue(_ggml_backend_graph_compute_async( + backend, + cgraph, + )); } - late final _ggml_op_symbolPtr = _lookup< - ffi.NativeFunction Function(ffi.UnsignedInt)>>( - 'ggml_op_symbol'); - late final _ggml_op_symbol = - _ggml_op_symbolPtr.asFunction Function(int)>(); + late final _ggml_backend_graph_compute_asyncPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ggml_backend_t, + ffi.Pointer)>>('ggml_backend_graph_compute_async'); + late final _ggml_backend_graph_compute_async = + _ggml_backend_graph_compute_asyncPtr + .asFunction)>(); - ffi.Pointer ggml_unary_op_name( - ggml_unary_op op, + ggml_backend_graph_copy$1 ggml_backend_graph_copy( + ggml_backend_t backend, + ffi.Pointer graph, ) { - return _ggml_unary_op_name( - op.value, + return _ggml_backend_graph_copy( + backend, + graph, ); } - late final _ggml_unary_op_namePtr = _lookup< - ffi.NativeFunction Function(ffi.UnsignedInt)>>( - 'ggml_unary_op_name'); - late final _ggml_unary_op_name = - _ggml_unary_op_namePtr.asFunction Function(int)>(); + late final _ggml_backend_graph_copyPtr = _lookup< + ffi.NativeFunction< + ggml_backend_graph_copy$1 Function(ggml_backend_t, + ffi.Pointer)>>('ggml_backend_graph_copy'); + late final _ggml_backend_graph_copy = _ggml_backend_graph_copyPtr.asFunction< + ggml_backend_graph_copy$1 Function( + ggml_backend_t, ffi.Pointer)>(); - ffi.Pointer ggml_glu_op_name( - ggml_glu_op op, + void ggml_backend_graph_copy_free( + ggml_backend_graph_copy$1 copy, ) { - return _ggml_glu_op_name( - op.value, + return _ggml_backend_graph_copy_free( + copy, ); } - late final _ggml_glu_op_namePtr = _lookup< - ffi.NativeFunction Function(ffi.UnsignedInt)>>( - 'ggml_glu_op_name'); - late final _ggml_glu_op_name = - _ggml_glu_op_namePtr.asFunction Function(int)>(); + late final _ggml_backend_graph_copy_freePtr = + _lookup>( + 'ggml_backend_graph_copy_free'); + late final _ggml_backend_graph_copy_free = _ggml_backend_graph_copy_freePtr + .asFunction(); - ffi.Pointer ggml_op_desc( - ffi.Pointer t, + ggml_status ggml_backend_graph_plan_compute( + ggml_backend_t backend, + ggml_backend_graph_plan_t plan, ) { - return _ggml_op_desc( - t, - ); + return ggml_status.fromValue(_ggml_backend_graph_plan_compute( + backend, + plan, + )); } - late final _ggml_op_descPtr = _lookup< + late final _ggml_backend_graph_plan_computePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('ggml_op_desc'); - late final _ggml_op_desc = _ggml_op_descPtr - .asFunction Function(ffi.Pointer)>(); + ffi.Int Function(ggml_backend_t, + ggml_backend_graph_plan_t)>>('ggml_backend_graph_plan_compute'); + late final _ggml_backend_graph_plan_compute = + _ggml_backend_graph_plan_computePtr.asFunction< + int Function(ggml_backend_t, ggml_backend_graph_plan_t)>(); - int ggml_element_size( - ffi.Pointer tensor, + ggml_backend_graph_plan_t ggml_backend_graph_plan_create( + ggml_backend_t backend, + ffi.Pointer cgraph, ) { - return _ggml_element_size( - tensor, + return _ggml_backend_graph_plan_create( + backend, + cgraph, ); } - late final _ggml_element_sizePtr = - _lookup)>>( - 'ggml_element_size'); - late final _ggml_element_size = _ggml_element_sizePtr - .asFunction)>(); + late final _ggml_backend_graph_plan_createPtr = _lookup< + ffi.NativeFunction< + ggml_backend_graph_plan_t Function(ggml_backend_t, + ffi.Pointer)>>('ggml_backend_graph_plan_create'); + late final _ggml_backend_graph_plan_create = + _ggml_backend_graph_plan_createPtr.asFunction< + ggml_backend_graph_plan_t Function( + ggml_backend_t, ffi.Pointer)>(); - bool ggml_is_quantized( - ggml_type type, + void ggml_backend_graph_plan_free( + ggml_backend_t backend, + ggml_backend_graph_plan_t plan, ) { - return _ggml_is_quantized( - type.value, + return _ggml_backend_graph_plan_free( + backend, + plan, ); } - late final _ggml_is_quantizedPtr = - _lookup>( - 'ggml_is_quantized'); - late final _ggml_is_quantized = - _ggml_is_quantizedPtr.asFunction(); + late final _ggml_backend_graph_plan_freePtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ggml_backend_t, + ggml_backend_graph_plan_t)>>('ggml_backend_graph_plan_free'); + late final _ggml_backend_graph_plan_free = _ggml_backend_graph_plan_freePtr + .asFunction(); - ggml_type ggml_ftype_to_ggml_type( - ggml_ftype ftype, + ggml_guid_t ggml_backend_guid( + ggml_backend_t backend, ) { - return ggml_type.fromValue(_ggml_ftype_to_ggml_type( - ftype.value, - )); + return _ggml_backend_guid( + backend, + ); } - late final _ggml_ftype_to_ggml_typePtr = - _lookup>( - 'ggml_ftype_to_ggml_type'); - late final _ggml_ftype_to_ggml_type = - _ggml_ftype_to_ggml_typePtr.asFunction(); + late final _ggml_backend_guidPtr = + _lookup>( + 'ggml_backend_guid'); + late final _ggml_backend_guid = + _ggml_backend_guidPtr.asFunction(); - bool ggml_is_transposed( - ffi.Pointer tensor, - ) { - return _ggml_is_transposed( - tensor, - ); + ggml_backend_t ggml_backend_init_best() { + return _ggml_backend_init_best(); } - late final _ggml_is_transposedPtr = - _lookup)>>( - 'ggml_is_transposed'); - late final _ggml_is_transposed = _ggml_is_transposedPtr - .asFunction)>(); + late final _ggml_backend_init_bestPtr = + _lookup>( + 'ggml_backend_init_best'); + late final _ggml_backend_init_best = + _ggml_backend_init_bestPtr.asFunction(); - bool ggml_is_permuted( - ffi.Pointer tensor, + ggml_backend_t ggml_backend_init_by_name( + ffi.Pointer name, + ffi.Pointer params, ) { - return _ggml_is_permuted( - tensor, + return _ggml_backend_init_by_name( + name, + params, ); } - late final _ggml_is_permutedPtr = - _lookup)>>( - 'ggml_is_permuted'); - late final _ggml_is_permuted = _ggml_is_permutedPtr - .asFunction)>(); + late final _ggml_backend_init_by_namePtr = _lookup< + ffi.NativeFunction< + ggml_backend_t Function(ffi.Pointer, + ffi.Pointer)>>('ggml_backend_init_by_name'); + late final _ggml_backend_init_by_name = + _ggml_backend_init_by_namePtr.asFunction< + ggml_backend_t Function( + ffi.Pointer, ffi.Pointer)>(); - bool ggml_is_empty( - ffi.Pointer tensor, + ggml_backend_t ggml_backend_init_by_type( + ggml_backend_dev_type type, + ffi.Pointer params, ) { - return _ggml_is_empty( - tensor, + return _ggml_backend_init_by_type( + type.value, + params, ); } - late final _ggml_is_emptyPtr = - _lookup)>>( - 'ggml_is_empty'); - late final _ggml_is_empty = - _ggml_is_emptyPtr.asFunction)>(); + late final _ggml_backend_init_by_typePtr = _lookup< + ffi.NativeFunction< + ggml_backend_t Function(ffi.UnsignedInt, + ffi.Pointer)>>('ggml_backend_init_by_type'); + late final _ggml_backend_init_by_type = _ggml_backend_init_by_typePtr + .asFunction)>(); - bool ggml_is_view( - ffi.Pointer tensor, + bool ggml_backend_is_cpu( + ggml_backend_t backend, ) { - return _ggml_is_view( - tensor, + return _ggml_backend_is_cpu( + backend, ); } - late final _ggml_is_viewPtr = - _lookup)>>( - 'ggml_is_view'); - late final _ggml_is_view = - _ggml_is_viewPtr.asFunction)>(); + late final _ggml_backend_is_cpuPtr = + _lookup>( + 'ggml_backend_is_cpu'); + late final _ggml_backend_is_cpu = + _ggml_backend_is_cpuPtr.asFunction(); - bool ggml_is_scalar( - ffi.Pointer tensor, + ggml_backend_reg_t ggml_backend_load( + ffi.Pointer path, ) { - return _ggml_is_scalar( - tensor, + return _ggml_backend_load( + path, ); } - late final _ggml_is_scalarPtr = - _lookup)>>( - 'ggml_is_scalar'); - late final _ggml_is_scalar = - _ggml_is_scalarPtr.asFunction)>(); + late final _ggml_backend_loadPtr = _lookup< + ffi + .NativeFunction)>>( + 'ggml_backend_load'); + late final _ggml_backend_load = _ggml_backend_loadPtr + .asFunction)>(); - bool ggml_is_vector( - ffi.Pointer tensor, + void ggml_backend_load_all() { + return _ggml_backend_load_all(); + } + + late final _ggml_backend_load_allPtr = + _lookup>('ggml_backend_load_all'); + late final _ggml_backend_load_all = + _ggml_backend_load_allPtr.asFunction(); + + void ggml_backend_load_all_from_path( + ffi.Pointer dir_path, ) { - return _ggml_is_vector( - tensor, + return _ggml_backend_load_all_from_path( + dir_path, ); } - late final _ggml_is_vectorPtr = - _lookup)>>( - 'ggml_is_vector'); - late final _ggml_is_vector = - _ggml_is_vectorPtr.asFunction)>(); + late final _ggml_backend_load_all_from_pathPtr = + _lookup)>>( + 'ggml_backend_load_all_from_path'); + late final _ggml_backend_load_all_from_path = + _ggml_backend_load_all_from_pathPtr + .asFunction)>(); - bool ggml_is_matrix( - ffi.Pointer tensor, + ggml_backend_dev_t ggml_backend_meta_device( + ffi.Pointer devs, + int n_devs, + ggml_backend_meta_get_split_state_t get_split_state, + ffi.Pointer get_split_state_ud, ) { - return _ggml_is_matrix( - tensor, + return _ggml_backend_meta_device( + devs, + n_devs, + get_split_state, + get_split_state_ud, ); } - late final _ggml_is_matrixPtr = - _lookup)>>( - 'ggml_is_matrix'); - late final _ggml_is_matrix = - _ggml_is_matrixPtr.asFunction)>(); + late final _ggml_backend_meta_devicePtr = _lookup< + ffi.NativeFunction< + ggml_backend_dev_t Function( + ffi.Pointer, + ffi.Size, + ggml_backend_meta_get_split_state_t, + ffi.Pointer)>>('ggml_backend_meta_device'); + late final _ggml_backend_meta_device = + _ggml_backend_meta_devicePtr.asFunction< + ggml_backend_dev_t Function(ffi.Pointer, int, + ggml_backend_meta_get_split_state_t, ffi.Pointer)>(); - bool ggml_is_3d( - ffi.Pointer tensor, + ffi.Pointer ggml_backend_meta_split_axis_name( + ggml_backend_meta_split_axis split_axis, ) { - return _ggml_is_3d( - tensor, + return _ggml_backend_meta_split_axis_name( + split_axis.value, ); } - late final _ggml_is_3dPtr = - _lookup)>>( - 'ggml_is_3d'); - late final _ggml_is_3d = - _ggml_is_3dPtr.asFunction)>(); + late final _ggml_backend_meta_split_axis_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'ggml_backend_meta_split_axis_name'); + late final _ggml_backend_meta_split_axis_name = + _ggml_backend_meta_split_axis_namePtr + .asFunction Function(int)>(); - int ggml_n_dims( - ffi.Pointer tensor, + ffi.Pointer ggml_backend_name( + ggml_backend_t backend, ) { - return _ggml_n_dims( - tensor, + return _ggml_backend_name( + backend, ); } - late final _ggml_n_dimsPtr = - _lookup)>>( - 'ggml_n_dims'); - late final _ggml_n_dims = - _ggml_n_dimsPtr.asFunction)>(); + late final _ggml_backend_namePtr = _lookup< + ffi.NativeFunction Function(ggml_backend_t)>>( + 'ggml_backend_name'); + late final _ggml_backend_name = _ggml_backend_namePtr + .asFunction Function(ggml_backend_t)>(); - bool ggml_is_contiguous( - ffi.Pointer tensor, + bool ggml_backend_offload_op( + ggml_backend_t backend, + ffi.Pointer op, ) { - return _ggml_is_contiguous( - tensor, + return _ggml_backend_offload_op( + backend, + op, ); } - late final _ggml_is_contiguousPtr = - _lookup)>>( - 'ggml_is_contiguous'); - late final _ggml_is_contiguous = _ggml_is_contiguousPtr - .asFunction)>(); + late final _ggml_backend_offload_opPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ggml_backend_t, + ffi.Pointer)>>('ggml_backend_offload_op'); + late final _ggml_backend_offload_op = _ggml_backend_offload_opPtr + .asFunction)>(); - bool ggml_is_contiguous_0( - ffi.Pointer tensor, + ggml_backend_reg_t ggml_backend_reg_by_name( + ffi.Pointer name, ) { - return _ggml_is_contiguous_0( - tensor, + return _ggml_backend_reg_by_name( + name, ); } - late final _ggml_is_contiguous_0Ptr = - _lookup)>>( - 'ggml_is_contiguous_0'); - late final _ggml_is_contiguous_0 = _ggml_is_contiguous_0Ptr - .asFunction)>(); + late final _ggml_backend_reg_by_namePtr = _lookup< + ffi + .NativeFunction)>>( + 'ggml_backend_reg_by_name'); + late final _ggml_backend_reg_by_name = _ggml_backend_reg_by_namePtr + .asFunction)>(); - bool ggml_is_contiguous_1( - ffi.Pointer tensor, - ) { - return _ggml_is_contiguous_1( - tensor, - ); + int ggml_backend_reg_count() { + return _ggml_backend_reg_count(); } - late final _ggml_is_contiguous_1Ptr = - _lookup)>>( - 'ggml_is_contiguous_1'); - late final _ggml_is_contiguous_1 = _ggml_is_contiguous_1Ptr - .asFunction)>(); + late final _ggml_backend_reg_countPtr = + _lookup>( + 'ggml_backend_reg_count'); + late final _ggml_backend_reg_count = + _ggml_backend_reg_countPtr.asFunction(); - bool ggml_is_contiguous_2( - ffi.Pointer tensor, + int ggml_backend_reg_dev_count( + ggml_backend_reg_t reg, ) { - return _ggml_is_contiguous_2( - tensor, + return _ggml_backend_reg_dev_count( + reg, ); } - late final _ggml_is_contiguous_2Ptr = - _lookup)>>( - 'ggml_is_contiguous_2'); - late final _ggml_is_contiguous_2 = _ggml_is_contiguous_2Ptr - .asFunction)>(); + late final _ggml_backend_reg_dev_countPtr = + _lookup>( + 'ggml_backend_reg_dev_count'); + late final _ggml_backend_reg_dev_count = _ggml_backend_reg_dev_countPtr + .asFunction(); - bool ggml_is_contiguously_allocated( - ffi.Pointer tensor, + ggml_backend_dev_t ggml_backend_reg_dev_get( + ggml_backend_reg_t reg, + int index, ) { - return _ggml_is_contiguously_allocated( - tensor, + return _ggml_backend_reg_dev_get( + reg, + index, ); } - late final _ggml_is_contiguously_allocatedPtr = - _lookup)>>( - 'ggml_is_contiguously_allocated'); - late final _ggml_is_contiguously_allocated = - _ggml_is_contiguously_allocatedPtr - .asFunction)>(); + late final _ggml_backend_reg_dev_getPtr = _lookup< + ffi.NativeFunction< + ggml_backend_dev_t Function( + ggml_backend_reg_t, ffi.Size)>>('ggml_backend_reg_dev_get'); + late final _ggml_backend_reg_dev_get = _ggml_backend_reg_dev_getPtr + .asFunction(); - bool ggml_is_contiguous_channels( - ffi.Pointer tensor, + ggml_backend_reg_t ggml_backend_reg_get( + int index, ) { - return _ggml_is_contiguous_channels( - tensor, + return _ggml_backend_reg_get( + index, ); } - late final _ggml_is_contiguous_channelsPtr = - _lookup)>>( - 'ggml_is_contiguous_channels'); - late final _ggml_is_contiguous_channels = _ggml_is_contiguous_channelsPtr - .asFunction)>(); + late final _ggml_backend_reg_getPtr = + _lookup>( + 'ggml_backend_reg_get'); + late final _ggml_backend_reg_get = + _ggml_backend_reg_getPtr.asFunction(); - bool ggml_is_contiguous_rows( - ffi.Pointer tensor, + ffi.Pointer ggml_backend_reg_get_proc_address( + ggml_backend_reg_t reg, + ffi.Pointer name, ) { - return _ggml_is_contiguous_rows( - tensor, + return _ggml_backend_reg_get_proc_address( + reg, + name, ); } - late final _ggml_is_contiguous_rowsPtr = - _lookup)>>( - 'ggml_is_contiguous_rows'); - late final _ggml_is_contiguous_rows = _ggml_is_contiguous_rowsPtr - .asFunction)>(); + late final _ggml_backend_reg_get_proc_addressPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ggml_backend_reg_t, + ffi.Pointer)>>('ggml_backend_reg_get_proc_address'); + late final _ggml_backend_reg_get_proc_address = + _ggml_backend_reg_get_proc_addressPtr.asFunction< + ffi.Pointer Function( + ggml_backend_reg_t, ffi.Pointer)>(); - bool ggml_are_same_shape( - ffi.Pointer t0, - ffi.Pointer t1, + ffi.Pointer ggml_backend_reg_name( + ggml_backend_reg_t reg, ) { - return _ggml_are_same_shape( - t0, - t1, + return _ggml_backend_reg_name( + reg, ); } - late final _ggml_are_same_shapePtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, - ffi.Pointer)>>('ggml_are_same_shape'); - late final _ggml_are_same_shape = _ggml_are_same_shapePtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer)>(); + late final _ggml_backend_reg_namePtr = _lookup< + ffi + .NativeFunction Function(ggml_backend_reg_t)>>( + 'ggml_backend_reg_name'); + late final _ggml_backend_reg_name = _ggml_backend_reg_namePtr + .asFunction Function(ggml_backend_reg_t)>(); - bool ggml_are_same_stride( - ffi.Pointer t0, - ffi.Pointer t1, + void ggml_backend_register( + ggml_backend_reg_t reg, ) { - return _ggml_are_same_stride( - t0, - t1, + return _ggml_backend_register( + reg, ); } - late final _ggml_are_same_stridePtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, - ffi.Pointer)>>('ggml_are_same_stride'); - late final _ggml_are_same_stride = _ggml_are_same_stridePtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer)>(); + late final _ggml_backend_registerPtr = + _lookup>( + 'ggml_backend_register'); + late final _ggml_backend_register = + _ggml_backend_registerPtr.asFunction(); - bool ggml_can_repeat( - ffi.Pointer t0, - ffi.Pointer t1, + bool ggml_backend_sched_alloc_graph( + ggml_backend_sched_t sched, + ffi.Pointer graph, ) { - return _ggml_can_repeat( - t0, - t1, + return _ggml_backend_sched_alloc_graph( + sched, + graph, ); } - late final _ggml_can_repeatPtr = _lookup< + late final _ggml_backend_sched_alloc_graphPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, - ffi.Pointer)>>('ggml_can_repeat'); - late final _ggml_can_repeat = _ggml_can_repeatPtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Bool Function(ggml_backend_sched_t, + ffi.Pointer)>>('ggml_backend_sched_alloc_graph'); + late final _ggml_backend_sched_alloc_graph = + _ggml_backend_sched_alloc_graphPtr.asFunction< + bool Function(ggml_backend_sched_t, ffi.Pointer)>(); - int ggml_tensor_overhead() { - return _ggml_tensor_overhead(); + void ggml_backend_sched_free( + ggml_backend_sched_t sched, + ) { + return _ggml_backend_sched_free( + sched, + ); } - late final _ggml_tensor_overheadPtr = - _lookup>('ggml_tensor_overhead'); - late final _ggml_tensor_overhead = - _ggml_tensor_overheadPtr.asFunction(); + late final _ggml_backend_sched_freePtr = + _lookup>( + 'ggml_backend_sched_free'); + late final _ggml_backend_sched_free = _ggml_backend_sched_freePtr + .asFunction(); - bool ggml_validate_row_data( - ggml_type type, - ffi.Pointer data, - int nbytes, + ggml_backend_t ggml_backend_sched_get_backend( + ggml_backend_sched_t sched, + int i, ) { - return _ggml_validate_row_data( - type.value, - data, - nbytes, + return _ggml_backend_sched_get_backend( + sched, + i, ); } - late final _ggml_validate_row_dataPtr = _lookup< + late final _ggml_backend_sched_get_backendPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.UnsignedInt, ffi.Pointer, - ffi.Size)>>('ggml_validate_row_data'); - late final _ggml_validate_row_data = _ggml_validate_row_dataPtr - .asFunction, int)>(); + ggml_backend_t Function(ggml_backend_sched_t, + ffi.Int)>>('ggml_backend_sched_get_backend'); + late final _ggml_backend_sched_get_backend = + _ggml_backend_sched_get_backendPtr + .asFunction(); - ffi.Pointer ggml_init( - ggml_init_params params, + int ggml_backend_sched_get_buffer_size( + ggml_backend_sched_t sched, + ggml_backend_t backend, ) { - return _ggml_init( - params, + return _ggml_backend_sched_get_buffer_size( + sched, + backend, ); } - late final _ggml_initPtr = _lookup< + late final _ggml_backend_sched_get_buffer_sizePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ggml_init_params)>>('ggml_init'); - late final _ggml_init = _ggml_initPtr - .asFunction Function(ggml_init_params)>(); + ffi.Size Function(ggml_backend_sched_t, + ggml_backend_t)>>('ggml_backend_sched_get_buffer_size'); + late final _ggml_backend_sched_get_buffer_size = + _ggml_backend_sched_get_buffer_sizePtr + .asFunction(); - void ggml_reset( - ffi.Pointer ctx, + ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type( + ggml_backend_sched_t sched, + ggml_backend_t backend, ) { - return _ggml_reset( - ctx, + return _ggml_backend_sched_get_buffer_type( + sched, + backend, ); } - late final _ggml_resetPtr = - _lookup)>>( - 'ggml_reset'); - late final _ggml_reset = - _ggml_resetPtr.asFunction)>(); - - void ggml_free( - ffi.Pointer ctx, - ) { - return _ggml_free( - ctx, - ); - } - - late final _ggml_freePtr = - _lookup)>>( - 'ggml_free'); - late final _ggml_free = - _ggml_freePtr.asFunction)>(); + late final _ggml_backend_sched_get_buffer_typePtr = _lookup< + ffi.NativeFunction< + ggml_backend_buffer_type_t Function(ggml_backend_sched_t, + ggml_backend_t)>>('ggml_backend_sched_get_buffer_type'); + late final _ggml_backend_sched_get_buffer_type = + _ggml_backend_sched_get_buffer_typePtr.asFunction< + ggml_backend_buffer_type_t Function( + ggml_backend_sched_t, ggml_backend_t)>(); - int ggml_used_mem( - ffi.Pointer ctx, + int ggml_backend_sched_get_n_backends( + ggml_backend_sched_t sched, ) { - return _ggml_used_mem( - ctx, + return _ggml_backend_sched_get_n_backends( + sched, ); } - late final _ggml_used_memPtr = - _lookup)>>( - 'ggml_used_mem'); - late final _ggml_used_mem = - _ggml_used_memPtr.asFunction)>(); + late final _ggml_backend_sched_get_n_backendsPtr = + _lookup>( + 'ggml_backend_sched_get_n_backends'); + late final _ggml_backend_sched_get_n_backends = + _ggml_backend_sched_get_n_backendsPtr + .asFunction(); - bool ggml_get_no_alloc( - ffi.Pointer ctx, + int ggml_backend_sched_get_n_copies( + ggml_backend_sched_t sched, ) { - return _ggml_get_no_alloc( - ctx, + return _ggml_backend_sched_get_n_copies( + sched, ); } - late final _ggml_get_no_allocPtr = - _lookup)>>( - 'ggml_get_no_alloc'); - late final _ggml_get_no_alloc = _ggml_get_no_allocPtr - .asFunction)>(); + late final _ggml_backend_sched_get_n_copiesPtr = + _lookup>( + 'ggml_backend_sched_get_n_copies'); + late final _ggml_backend_sched_get_n_copies = + _ggml_backend_sched_get_n_copiesPtr + .asFunction(); - void ggml_set_no_alloc( - ffi.Pointer ctx, - bool no_alloc, + int ggml_backend_sched_get_n_splits( + ggml_backend_sched_t sched, ) { - return _ggml_set_no_alloc( - ctx, - no_alloc, + return _ggml_backend_sched_get_n_splits( + sched, ); } - late final _ggml_set_no_allocPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, ffi.Bool)>>('ggml_set_no_alloc'); - late final _ggml_set_no_alloc = _ggml_set_no_allocPtr - .asFunction, bool)>(); + late final _ggml_backend_sched_get_n_splitsPtr = + _lookup>( + 'ggml_backend_sched_get_n_splits'); + late final _ggml_backend_sched_get_n_splits = + _ggml_backend_sched_get_n_splitsPtr + .asFunction(); - ffi.Pointer ggml_get_mem_buffer( - ffi.Pointer ctx, + ggml_backend_t ggml_backend_sched_get_tensor_backend( + ggml_backend_sched_t sched, + ffi.Pointer node, ) { - return _ggml_get_mem_buffer( - ctx, + return _ggml_backend_sched_get_tensor_backend( + sched, + node, ); } - late final _ggml_get_mem_bufferPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('ggml_get_mem_buffer'); - late final _ggml_get_mem_buffer = _ggml_get_mem_bufferPtr - .asFunction Function(ffi.Pointer)>(); + late final _ggml_backend_sched_get_tensor_backendPtr = _lookup< + ffi.NativeFunction< + ggml_backend_t Function( + ggml_backend_sched_t, ffi.Pointer)>>( + 'ggml_backend_sched_get_tensor_backend'); + late final _ggml_backend_sched_get_tensor_backend = + _ggml_backend_sched_get_tensor_backendPtr.asFunction< + ggml_backend_t Function( + ggml_backend_sched_t, ffi.Pointer)>(); - int ggml_get_mem_size( - ffi.Pointer ctx, + ggml_status ggml_backend_sched_graph_compute( + ggml_backend_sched_t sched, + ffi.Pointer graph, ) { - return _ggml_get_mem_size( - ctx, - ); + return ggml_status.fromValue(_ggml_backend_sched_graph_compute( + sched, + graph, + )); } - late final _ggml_get_mem_sizePtr = - _lookup)>>( - 'ggml_get_mem_size'); - late final _ggml_get_mem_size = _ggml_get_mem_sizePtr - .asFunction)>(); + late final _ggml_backend_sched_graph_computePtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ggml_backend_sched_t, + ffi.Pointer)>>('ggml_backend_sched_graph_compute'); + late final _ggml_backend_sched_graph_compute = + _ggml_backend_sched_graph_computePtr.asFunction< + int Function(ggml_backend_sched_t, ffi.Pointer)>(); - int ggml_get_max_tensor_size( - ffi.Pointer ctx, + ggml_status ggml_backend_sched_graph_compute_async( + ggml_backend_sched_t sched, + ffi.Pointer graph, ) { - return _ggml_get_max_tensor_size( - ctx, - ); + return ggml_status.fromValue(_ggml_backend_sched_graph_compute_async( + sched, + graph, + )); } - late final _ggml_get_max_tensor_sizePtr = - _lookup)>>( - 'ggml_get_max_tensor_size'); - late final _ggml_get_max_tensor_size = _ggml_get_max_tensor_sizePtr - .asFunction)>(); + late final _ggml_backend_sched_graph_compute_asyncPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function( + ggml_backend_sched_t, ffi.Pointer)>>( + 'ggml_backend_sched_graph_compute_async'); + late final _ggml_backend_sched_graph_compute_async = + _ggml_backend_sched_graph_compute_asyncPtr.asFunction< + int Function(ggml_backend_sched_t, ffi.Pointer)>(); - ffi.Pointer ggml_new_tensor( - ffi.Pointer ctx, - ggml_type type, - int n_dims, - ffi.Pointer ne, + ggml_backend_sched_t ggml_backend_sched_new( + ffi.Pointer backends, + ffi.Pointer bufts, + int n_backends, + int graph_size, + bool parallel, + bool op_offload, ) { - return _ggml_new_tensor( - ctx, - type.value, - n_dims, - ne, + return _ggml_backend_sched_new( + backends, + bufts, + n_backends, + graph_size, + parallel, + op_offload, ); } - late final _ggml_new_tensorPtr = _lookup< + late final _ggml_backend_sched_newPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.UnsignedInt, + ggml_backend_sched_t Function( + ffi.Pointer, + ffi.Pointer, ffi.Int, - ffi.Pointer)>>('ggml_new_tensor'); - late final _ggml_new_tensor = _ggml_new_tensorPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int, int, ffi.Pointer)>(); + ffi.Size, + ffi.Bool, + ffi.Bool)>>('ggml_backend_sched_new'); + late final _ggml_backend_sched_new = _ggml_backend_sched_newPtr.asFunction< + ggml_backend_sched_t Function(ffi.Pointer, + ffi.Pointer, int, int, bool, bool)>(); - ffi.Pointer ggml_new_tensor_1d( - ffi.Pointer ctx, - ggml_type type, - int ne0, + bool ggml_backend_sched_reserve( + ggml_backend_sched_t sched, + ffi.Pointer measure_graph, ) { - return _ggml_new_tensor_1d( - ctx, - type.value, - ne0, + return _ggml_backend_sched_reserve( + sched, + measure_graph, ); } - late final _ggml_new_tensor_1dPtr = _lookup< + late final _ggml_backend_sched_reservePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.UnsignedInt, ffi.Int64)>>('ggml_new_tensor_1d'); - late final _ggml_new_tensor_1d = _ggml_new_tensor_1dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int, int)>(); + ffi.Bool Function(ggml_backend_sched_t, + ffi.Pointer)>>('ggml_backend_sched_reserve'); + late final _ggml_backend_sched_reserve = + _ggml_backend_sched_reservePtr.asFunction< + bool Function(ggml_backend_sched_t, ffi.Pointer)>(); - ffi.Pointer ggml_new_tensor_2d( - ffi.Pointer ctx, - ggml_type type, - int ne0, - int ne1, + void ggml_backend_sched_reserve_size( + ggml_backend_sched_t sched, + ffi.Pointer measure_graph, + ffi.Pointer sizes, ) { - return _ggml_new_tensor_2d( - ctx, - type.value, - ne0, - ne1, + return _ggml_backend_sched_reserve_size( + sched, + measure_graph, + sizes, ); } - late final _ggml_new_tensor_2dPtr = _lookup< + late final _ggml_backend_sched_reserve_sizePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.UnsignedInt, ffi.Int64, ffi.Int64)>>('ggml_new_tensor_2d'); - late final _ggml_new_tensor_2d = _ggml_new_tensor_2dPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int, int, int)>(); + ffi.Void Function(ggml_backend_sched_t, ffi.Pointer, + ffi.Pointer)>>('ggml_backend_sched_reserve_size'); + late final _ggml_backend_sched_reserve_size = + _ggml_backend_sched_reserve_sizePtr.asFunction< + void Function(ggml_backend_sched_t, ffi.Pointer, + ffi.Pointer)>(); - ffi.Pointer ggml_new_tensor_3d( - ffi.Pointer ctx, - ggml_type type, - int ne0, - int ne1, - int ne2, + void ggml_backend_sched_reset( + ggml_backend_sched_t sched, ) { - return _ggml_new_tensor_3d( - ctx, - type.value, - ne0, - ne1, - ne2, + return _ggml_backend_sched_reset( + sched, ); } - late final _ggml_new_tensor_3dPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.UnsignedInt, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_new_tensor_3d'); - late final _ggml_new_tensor_3d = _ggml_new_tensor_3dPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int, int, int, int)>(); + late final _ggml_backend_sched_resetPtr = + _lookup>( + 'ggml_backend_sched_reset'); + late final _ggml_backend_sched_reset = _ggml_backend_sched_resetPtr + .asFunction(); - ffi.Pointer ggml_new_tensor_4d( - ffi.Pointer ctx, - ggml_type type, - int ne0, - int ne1, - int ne2, - int ne3, + void ggml_backend_sched_set_eval_callback( + ggml_backend_sched_t sched, + ggml_backend_sched_eval_callback callback, + ffi.Pointer user_data, ) { - return _ggml_new_tensor_4d( - ctx, - type.value, - ne0, - ne1, - ne2, - ne3, + return _ggml_backend_sched_set_eval_callback( + sched, + callback, + user_data, ); } - late final _ggml_new_tensor_4dPtr = _lookup< + late final _ggml_backend_sched_set_eval_callbackPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.UnsignedInt, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_new_tensor_4d'); - late final _ggml_new_tensor_4d = _ggml_new_tensor_4dPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int, int, int, int, int)>(); + ffi.Void Function( + ggml_backend_sched_t, + ggml_backend_sched_eval_callback, + ffi.Pointer)>>('ggml_backend_sched_set_eval_callback'); + late final _ggml_backend_sched_set_eval_callback = + _ggml_backend_sched_set_eval_callbackPtr.asFunction< + void Function(ggml_backend_sched_t, ggml_backend_sched_eval_callback, + ffi.Pointer)>(); - ffi.Pointer ggml_new_buffer( - ffi.Pointer ctx, - int nbytes, + void ggml_backend_sched_set_tensor_backend( + ggml_backend_sched_t sched, + ffi.Pointer node, + ggml_backend_t backend, ) { - return _ggml_new_buffer( - ctx, - nbytes, + return _ggml_backend_sched_set_tensor_backend( + sched, + node, + backend, ); } - late final _ggml_new_bufferPtr = _lookup< + late final _ggml_backend_sched_set_tensor_backendPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Size)>>('ggml_new_buffer'); - late final _ggml_new_buffer = _ggml_new_bufferPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Void Function(ggml_backend_sched_t, ffi.Pointer, + ggml_backend_t)>>('ggml_backend_sched_set_tensor_backend'); + late final _ggml_backend_sched_set_tensor_backend = + _ggml_backend_sched_set_tensor_backendPtr.asFunction< + void Function(ggml_backend_sched_t, ffi.Pointer, + ggml_backend_t)>(); - ffi.Pointer ggml_dup_tensor( - ffi.Pointer ctx, - ffi.Pointer src, + void ggml_backend_sched_split_graph( + ggml_backend_sched_t sched, + ffi.Pointer graph, ) { - return _ggml_dup_tensor( - ctx, - src, + return _ggml_backend_sched_split_graph( + sched, + graph, ); } - late final _ggml_dup_tensorPtr = _lookup< + late final _ggml_backend_sched_split_graphPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_dup_tensor'); - late final _ggml_dup_tensor = _ggml_dup_tensorPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ggml_backend_sched_t, + ffi.Pointer)>>('ggml_backend_sched_split_graph'); + late final _ggml_backend_sched_split_graph = + _ggml_backend_sched_split_graphPtr.asFunction< + void Function(ggml_backend_sched_t, ffi.Pointer)>(); - ffi.Pointer ggml_view_tensor( - ffi.Pointer ctx, - ffi.Pointer src, + void ggml_backend_sched_synchronize( + ggml_backend_sched_t sched, ) { - return _ggml_view_tensor( - ctx, - src, + return _ggml_backend_sched_synchronize( + sched, ); } - late final _ggml_view_tensorPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_view_tensor'); - late final _ggml_view_tensor = _ggml_view_tensorPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_backend_sched_synchronizePtr = + _lookup>( + 'ggml_backend_sched_synchronize'); + late final _ggml_backend_sched_synchronize = + _ggml_backend_sched_synchronizePtr + .asFunction(); - ffi.Pointer ggml_get_first_tensor( - ffi.Pointer ctx, + bool ggml_backend_supports_buft( + ggml_backend_t backend, + ggml_backend_buffer_type_t buft, ) { - return _ggml_get_first_tensor( - ctx, + return _ggml_backend_supports_buft( + backend, + buft, ); } - late final _ggml_get_first_tensorPtr = _lookup< + late final _ggml_backend_supports_buftPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('ggml_get_first_tensor'); - late final _ggml_get_first_tensor = _ggml_get_first_tensorPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + ffi.Bool Function(ggml_backend_t, + ggml_backend_buffer_type_t)>>('ggml_backend_supports_buft'); + late final _ggml_backend_supports_buft = _ggml_backend_supports_buftPtr + .asFunction(); - ffi.Pointer ggml_get_next_tensor( - ffi.Pointer ctx, - ffi.Pointer tensor, + bool ggml_backend_supports_op( + ggml_backend_t backend, + ffi.Pointer op, ) { - return _ggml_get_next_tensor( - ctx, - tensor, + return _ggml_backend_supports_op( + backend, + op, ); } - late final _ggml_get_next_tensorPtr = _lookup< + late final _ggml_backend_supports_opPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_get_next_tensor'); - late final _ggml_get_next_tensor = _ggml_get_next_tensorPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Bool Function(ggml_backend_t, + ffi.Pointer)>>('ggml_backend_supports_op'); + late final _ggml_backend_supports_op = _ggml_backend_supports_opPtr + .asFunction)>(); - ffi.Pointer ggml_get_tensor( - ffi.Pointer ctx, - ffi.Pointer name, + void ggml_backend_synchronize( + ggml_backend_t backend, ) { - return _ggml_get_tensor( - ctx, - name, + return _ggml_backend_synchronize( + backend, ); } - late final _ggml_get_tensorPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_get_tensor'); - late final _ggml_get_tensor = _ggml_get_tensorPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_backend_synchronizePtr = + _lookup>( + 'ggml_backend_synchronize'); + late final _ggml_backend_synchronize = + _ggml_backend_synchronizePtr.asFunction(); - void ggml_unravel_index( + ggml_status ggml_backend_tensor_alloc( + ggml_backend_buffer_t buffer, ffi.Pointer tensor, - int i, - ffi.Pointer i0, - ffi.Pointer i1, - ffi.Pointer i2, - ffi.Pointer i3, + ffi.Pointer addr, ) { - return _ggml_unravel_index( + return ggml_status.fromValue(_ggml_backend_tensor_alloc( + buffer, tensor, - i, - i0, - i1, - i2, - i3, - ); + addr, + )); } - late final _ggml_unravel_indexPtr = _lookup< + late final _ggml_backend_tensor_allocPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, - ffi.Int64, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_unravel_index'); - late final _ggml_unravel_index = _ggml_unravel_indexPtr.asFunction< - void Function( - ffi.Pointer, - int, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Int Function(ggml_backend_buffer_t, ffi.Pointer, + ffi.Pointer)>>('ggml_backend_tensor_alloc'); + late final _ggml_backend_tensor_alloc = + _ggml_backend_tensor_allocPtr.asFunction< + int Function(ggml_backend_buffer_t, ffi.Pointer, + ffi.Pointer)>(); - ggml_unary_op ggml_get_unary_op( - ffi.Pointer tensor, + void ggml_backend_tensor_copy( + ffi.Pointer src, + ffi.Pointer dst, ) { - return ggml_unary_op.fromValue(_ggml_get_unary_op( - tensor, - )); - } - - late final _ggml_get_unary_opPtr = _lookup< - ffi - .NativeFunction)>>( - 'ggml_get_unary_op'); - late final _ggml_get_unary_op = _ggml_get_unary_opPtr - .asFunction)>(); - - ggml_glu_op ggml_get_glu_op( - ffi.Pointer tensor, - ) { - return ggml_glu_op.fromValue(_ggml_get_glu_op( - tensor, - )); + return _ggml_backend_tensor_copy( + src, + dst, + ); } - late final _ggml_get_glu_opPtr = _lookup< - ffi - .NativeFunction)>>( - 'ggml_get_glu_op'); - late final _ggml_get_glu_op = - _ggml_get_glu_opPtr.asFunction)>(); + late final _ggml_backend_tensor_copyPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('ggml_backend_tensor_copy'); + late final _ggml_backend_tensor_copy = + _ggml_backend_tensor_copyPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_get_data( - ffi.Pointer tensor, + void ggml_backend_tensor_copy_async( + ggml_backend_t backend_src, + ggml_backend_t backend_dst, + ffi.Pointer src, + ffi.Pointer dst, ) { - return _ggml_get_data( - tensor, + return _ggml_backend_tensor_copy_async( + backend_src, + backend_dst, + src, + dst, ); } - late final _ggml_get_dataPtr = _lookup< + late final _ggml_backend_tensor_copy_asyncPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('ggml_get_data'); - late final _ggml_get_data = _ggml_get_dataPtr - .asFunction Function(ffi.Pointer)>(); + ffi.Void Function( + ggml_backend_t, + ggml_backend_t, + ffi.Pointer, + ffi.Pointer)>>('ggml_backend_tensor_copy_async'); + late final _ggml_backend_tensor_copy_async = + _ggml_backend_tensor_copy_asyncPtr.asFunction< + void Function(ggml_backend_t, ggml_backend_t, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_get_data_f32( + void ggml_backend_tensor_get( ffi.Pointer tensor, + ffi.Pointer data, + int offset, + int size, ) { - return _ggml_get_data_f32( + return _ggml_backend_tensor_get( tensor, + data, + offset, + size, ); } - late final _ggml_get_data_f32Ptr = _lookup< + late final _ggml_backend_tensor_getPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('ggml_get_data_f32'); - late final _ggml_get_data_f32 = _ggml_get_data_f32Ptr - .asFunction Function(ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Size, ffi.Size)>>('ggml_backend_tensor_get'); + late final _ggml_backend_tensor_get = _ggml_backend_tensor_getPtr.asFunction< + void Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - ffi.Pointer ggml_get_name( + void ggml_backend_tensor_get_2d( ffi.Pointer tensor, + ffi.Pointer data, + int offset, + int size, + int n_copies, + int stride_tensor, + int stride_data, ) { - return _ggml_get_name( + return _ggml_backend_tensor_get_2d( tensor, + data, + offset, + size, + n_copies, + stride_tensor, + stride_data, ); } - late final _ggml_get_namePtr = _lookup< + late final _ggml_backend_tensor_get_2dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('ggml_get_name'); - late final _ggml_get_name = _ggml_get_namePtr - .asFunction Function(ffi.Pointer)>(); + ffi.Void Function( + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_backend_tensor_get_2d'); + late final _ggml_backend_tensor_get_2d = + _ggml_backend_tensor_get_2dPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int, + int, int, int, int)>(); - ffi.Pointer ggml_set_name( + void ggml_backend_tensor_get_2d_async( + ggml_backend_t backend, ffi.Pointer tensor, - ffi.Pointer name, + ffi.Pointer data, + int offset, + int size, + int n_copies, + int stride_tensor, + int stride_data, ) { - return _ggml_set_name( + return _ggml_backend_tensor_get_2d_async( + backend, tensor, - name, + data, + offset, + size, + n_copies, + stride_tensor, + stride_data, ); } - late final _ggml_set_namePtr = _lookup< + late final _ggml_backend_tensor_get_2d_asyncPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_set_name'); - late final _ggml_set_name = _ggml_set_namePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function( + ggml_backend_t, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_backend_tensor_get_2d_async'); + late final _ggml_backend_tensor_get_2d_async = + _ggml_backend_tensor_get_2d_asyncPtr.asFunction< + void Function(ggml_backend_t, ffi.Pointer, + ffi.Pointer, int, int, int, int, int)>(); - ffi.Pointer ggml_format_name( + void ggml_backend_tensor_get_async( + ggml_backend_t backend, ffi.Pointer tensor, - ffi.Pointer fmt, + ffi.Pointer data, + int offset, + int size, ) { - return _ggml_format_name( + return _ggml_backend_tensor_get_async( + backend, tensor, - fmt, + data, + offset, + size, ); } - late final _ggml_format_namePtr = _lookup< + late final _ggml_backend_tensor_get_asyncPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_format_name'); - late final _ggml_format_name = _ggml_format_namePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function( + ggml_backend_t, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size)>>('ggml_backend_tensor_get_async'); + late final _ggml_backend_tensor_get_async = + _ggml_backend_tensor_get_asyncPtr.asFunction< + void Function(ggml_backend_t, ffi.Pointer, + ffi.Pointer, int, int)>(); - void ggml_set_input( + void ggml_backend_tensor_memset( ffi.Pointer tensor, + int value, + int offset, + int size, ) { - return _ggml_set_input( + return _ggml_backend_tensor_memset( tensor, + value, + offset, + size, ); } - late final _ggml_set_inputPtr = - _lookup)>>( - 'ggml_set_input'); - late final _ggml_set_input = - _ggml_set_inputPtr.asFunction)>(); + late final _ggml_backend_tensor_memsetPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Uint8, ffi.Size, + ffi.Size)>>('ggml_backend_tensor_memset'); + late final _ggml_backend_tensor_memset = _ggml_backend_tensor_memsetPtr + .asFunction, int, int, int)>(); - void ggml_set_output( + void ggml_backend_tensor_set( ffi.Pointer tensor, + ffi.Pointer data, + int offset, + int size, ) { - return _ggml_set_output( + return _ggml_backend_tensor_set( tensor, + data, + offset, + size, ); } - late final _ggml_set_outputPtr = - _lookup)>>( - 'ggml_set_output'); - late final _ggml_set_output = - _ggml_set_outputPtr.asFunction)>(); + late final _ggml_backend_tensor_setPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Size, ffi.Size)>>('ggml_backend_tensor_set'); + late final _ggml_backend_tensor_set = _ggml_backend_tensor_setPtr.asFunction< + void Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - void ggml_set_param( + void ggml_backend_tensor_set_2d( ffi.Pointer tensor, + ffi.Pointer data, + int offset, + int size, + int n_copies, + int stride_tensor, + int stride_data, ) { - return _ggml_set_param( + return _ggml_backend_tensor_set_2d( tensor, + data, + offset, + size, + n_copies, + stride_tensor, + stride_data, ); } - late final _ggml_set_paramPtr = - _lookup)>>( - 'ggml_set_param'); - late final _ggml_set_param = - _ggml_set_paramPtr.asFunction)>(); - - void ggml_set_loss( - ffi.Pointer tensor, - ) { - return _ggml_set_loss( - tensor, - ); - } - - late final _ggml_set_lossPtr = - _lookup)>>( - 'ggml_set_loss'); - late final _ggml_set_loss = - _ggml_set_lossPtr.asFunction)>(); + late final _ggml_backend_tensor_set_2dPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_backend_tensor_set_2d'); + late final _ggml_backend_tensor_set_2d = + _ggml_backend_tensor_set_2dPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int, + int, int, int, int)>(); - ffi.Pointer ggml_dup( - ffi.Pointer ctx, - ffi.Pointer a, + void ggml_backend_tensor_set_2d_async( + ggml_backend_t backend, + ffi.Pointer tensor, + ffi.Pointer data, + int offset, + int size, + int n_copies, + int stride_tensor, + int stride_data, ) { - return _ggml_dup( - ctx, - a, + return _ggml_backend_tensor_set_2d_async( + backend, + tensor, + data, + offset, + size, + n_copies, + stride_tensor, + stride_data, ); } - late final _ggml_dupPtr = _lookup< + late final _ggml_backend_tensor_set_2d_asyncPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_dup'); - late final _ggml_dup = _ggml_dupPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function( + ggml_backend_t, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_backend_tensor_set_2d_async'); + late final _ggml_backend_tensor_set_2d_async = + _ggml_backend_tensor_set_2d_asyncPtr.asFunction< + void Function(ggml_backend_t, ffi.Pointer, + ffi.Pointer, int, int, int, int, int)>(); - ffi.Pointer ggml_dup_inplace( - ffi.Pointer ctx, - ffi.Pointer a, + void ggml_backend_tensor_set_async( + ggml_backend_t backend, + ffi.Pointer tensor, + ffi.Pointer data, + int offset, + int size, ) { - return _ggml_dup_inplace( - ctx, - a, + return _ggml_backend_tensor_set_async( + backend, + tensor, + data, + offset, + size, ); } - late final _ggml_dup_inplacePtr = _lookup< + late final _ggml_backend_tensor_set_asyncPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_dup_inplace'); - late final _ggml_dup_inplace = _ggml_dup_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function( + ggml_backend_t, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size)>>('ggml_backend_tensor_set_async'); + late final _ggml_backend_tensor_set_async = + _ggml_backend_tensor_set_asyncPtr.asFunction< + void Function(ggml_backend_t, ffi.Pointer, + ffi.Pointer, int, int)>(); - ffi.Pointer ggml_add( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + void ggml_backend_unload( + ggml_backend_reg_t reg, ) { - return _ggml_add( - ctx, - a, - b, + return _ggml_backend_unload( + reg, ); } - late final _ggml_addPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>>('ggml_add'); - late final _ggml_add = _ggml_addPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_backend_unloadPtr = + _lookup>( + 'ggml_backend_unload'); + late final _ggml_backend_unload = + _ggml_backend_unloadPtr.asFunction(); - ffi.Pointer ggml_add_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + ggml_status ggml_backend_view_init( + ffi.Pointer tensor, ) { - return _ggml_add_inplace( - ctx, - a, - b, - ); + return ggml_status.fromValue(_ggml_backend_view_init( + tensor, + )); } - late final _ggml_add_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_add_inplace'); - late final _ggml_add_inplace = _ggml_add_inplacePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_backend_view_initPtr = + _lookup)>>( + 'ggml_backend_view_init'); + late final _ggml_backend_view_init = _ggml_backend_view_initPtr + .asFunction)>(); - ffi.Pointer ggml_add_cast( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ggml_type type, + double ggml_bf16_to_fp32( + ggml_bf16_t arg0, ) { - return _ggml_add_cast( - ctx, - a, - b, - type.value, + return _ggml_bf16_to_fp32( + arg0, ); } - late final _ggml_add_castPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.UnsignedInt)>>('ggml_add_cast'); - late final _ggml_add_cast = _ggml_add_castPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_bf16_to_fp32Ptr = + _lookup>( + 'ggml_bf16_to_fp32'); + late final _ggml_bf16_to_fp32 = + _ggml_bf16_to_fp32Ptr.asFunction(); - ffi.Pointer ggml_add_id( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer ids, + void ggml_bf16_to_fp32_row( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_add_id( - ctx, - a, - b, - ids, + return _ggml_bf16_to_fp32_row( + arg0, + arg1, + arg2, ); } - late final _ggml_add_idPtr = _lookup< + late final _ggml_bf16_to_fp32_rowPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_add_id'); - late final _ggml_add_id = _ggml_add_idPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_bf16_to_fp32_row'); + late final _ggml_bf16_to_fp32_row = _ggml_bf16_to_fp32_rowPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_add1( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + int ggml_blck_size( + ggml_type type, ) { - return _ggml_add1( - ctx, - a, - b, + return _ggml_blck_size( + type.value, ); } - late final _ggml_add1Ptr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_add1'); - late final _ggml_add1 = _ggml_add1Ptr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_blck_sizePtr = + _lookup>( + 'ggml_blck_size'); + late final _ggml_blck_size = + _ggml_blck_sizePtr.asFunction(); - ffi.Pointer ggml_add1_inplace( + void ggml_build_backward_expand( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + ffi.Pointer cgraph, + ffi.Pointer> grad_accs, ) { - return _ggml_add1_inplace( + return _ggml_build_backward_expand( ctx, - a, - b, + cgraph, + grad_accs, ); } - late final _ggml_add1_inplacePtr = _lookup< + late final _ggml_build_backward_expandPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>)>>( + 'ggml_build_backward_expand'); + late final _ggml_build_backward_expand = + _ggml_build_backward_expandPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer>)>(); + + void ggml_build_forward_expand( + ffi.Pointer cgraph, + ffi.Pointer tensor, + ) { + return _ggml_build_forward_expand( + cgraph, + tensor, + ); + } + + late final _ggml_build_forward_expandPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_add1_inplace'); - late final _ggml_add1_inplace = _ggml_add1_inplacePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('ggml_build_forward_expand'); + late final _ggml_build_forward_expand = + _ggml_build_forward_expandPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_acc( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int nb1, - int nb2, - int nb3, - int offset, + ffi.Pointer ggml_build_forward_select( + ffi.Pointer cgraph, + ffi.Pointer> tensors, + int n_tensors, + int idx, ) { - return _ggml_acc( - ctx, - a, - b, - nb1, - nb2, - nb3, - offset, + return _ggml_build_forward_select( + cgraph, + tensors, + n_tensors, + idx, ); } - late final _ggml_accPtr = _lookup< + late final _ggml_build_forward_selectPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_acc'); - late final _ggml_acc = _ggml_accPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int)>(); + ffi.Pointer, + ffi.Pointer>, + ffi.Int, + ffi.Int)>>('ggml_build_forward_select'); + late final _ggml_build_forward_select = + _ggml_build_forward_selectPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer>, int, int)>(); - ffi.Pointer ggml_acc_inplace( + bool ggml_can_repeat( + ffi.Pointer t0, + ffi.Pointer t1, + ) { + return _ggml_can_repeat( + t0, + t1, + ); + } + + late final _ggml_can_repeatPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, + ffi.Pointer)>>('ggml_can_repeat'); + late final _ggml_can_repeat = _ggml_can_repeatPtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer)>(); + + ffi.Pointer ggml_cast( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, - int nb1, - int nb2, - int nb3, - int offset, + ggml_type type, ) { - return _ggml_acc_inplace( + return _ggml_cast( ctx, a, - b, - nb1, - nb2, - nb3, - offset, + type.value, ); } - late final _ggml_acc_inplacePtr = _lookup< + late final _ggml_castPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_acc_inplace'); - late final _ggml_acc_inplace = _ggml_acc_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.UnsignedInt)>>('ggml_cast'); + late final _ggml_cast = _ggml_castPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int)>(); + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_sub( + ffi.Pointer ggml_ceil( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_sub( + return _ggml_ceil( ctx, a, - b, ); } - late final _ggml_subPtr = _lookup< + late final _ggml_ceilPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>>('ggml_sub'); - late final _ggml_sub = _ggml_subPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer)>>('ggml_ceil'); + late final _ggml_ceil = _ggml_ceilPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_sub_inplace( + ffi.Pointer ggml_ceil_inplace( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_sub_inplace( + return _ggml_ceil_inplace( ctx, a, - b, ); } - late final _ggml_sub_inplacePtr = _lookup< + late final _ggml_ceil_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_sub_inplace'); - late final _ggml_sub_inplace = _ggml_sub_inplacePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_ceil_inplace'); + late final _ggml_ceil_inplace = _ggml_ceil_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_mul( + ffi.Pointer ggml_clamp( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, + double min, + double max, ) { - return _ggml_mul( + return _ggml_clamp( ctx, a, - b, + min, + max, ); } - late final _ggml_mulPtr = _lookup< + late final _ggml_clampPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>>('ggml_mul'); - late final _ggml_mul = _ggml_mulPtr.asFunction< + ffi.Pointer, ffi.Float, ffi.Float)>>('ggml_clamp'); + late final _ggml_clamp = _ggml_clampPtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, double, double)>(); - ffi.Pointer ggml_mul_inplace( + ffi.Pointer ggml_col2im_1d( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, + int s0, + int oc, + int p0, ) { - return _ggml_mul_inplace( + return _ggml_col2im_1d( ctx, a, - b, + s0, + oc, + p0, ); } - late final _ggml_mul_inplacePtr = _lookup< + late final _ggml_col2im_1dPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('ggml_mul_inplace'); - late final _ggml_mul_inplace = _ggml_mul_inplacePtr.asFunction< + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_col2im_1d'); + late final _ggml_col2im_1d = _ggml_col2im_1dPtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, int, int, int)>(); - ffi.Pointer ggml_div( + ffi.Pointer ggml_commit() { + return _ggml_commit(); + } + + late final _ggml_commitPtr = + _lookup Function()>>( + 'ggml_commit'); + late final _ggml_commit = + _ggml_commitPtr.asFunction Function()>(); + + ffi.Pointer ggml_concat( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, + int dim, ) { - return _ggml_div( + return _ggml_concat( ctx, a, b, + dim, ); } - late final _ggml_divPtr = _lookup< + late final _ggml_concatPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>>('ggml_div'); - late final _ggml_div = _ggml_divPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int)>>('ggml_concat'); + late final _ggml_concat = _ggml_concatPtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_div_inplace( + ffi.Pointer ggml_cont( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_div_inplace( + return _ggml_cont( ctx, a, - b, ); } - late final _ggml_div_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_div_inplace'); - late final _ggml_div_inplace = _ggml_div_inplacePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); - - ffi.Pointer ggml_sqr( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_sqr( - ctx, - a, - ); - } - - late final _ggml_sqrPtr = _lookup< + late final _ggml_contPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sqr'); - late final _ggml_sqr = _ggml_sqrPtr.asFunction< + ffi.Pointer)>>('ggml_cont'); + late final _ggml_cont = _ggml_contPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_sqr_inplace( + ffi.Pointer ggml_cont_1d( ffi.Pointer ctx, ffi.Pointer a, + int ne0, ) { - return _ggml_sqr_inplace( + return _ggml_cont_1d( ctx, a, + ne0, ); } - late final _ggml_sqr_inplacePtr = _lookup< + late final _ggml_cont_1dPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sqr_inplace'); - late final _ggml_sqr_inplace = _ggml_sqr_inplacePtr.asFunction< + ffi.Pointer, ffi.Int64)>>('ggml_cont_1d'); + late final _ggml_cont_1d = _ggml_cont_1dPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_sqrt( + ffi.Pointer ggml_cont_2d( ffi.Pointer ctx, ffi.Pointer a, + int ne0, + int ne1, ) { - return _ggml_sqrt( + return _ggml_cont_2d( ctx, a, + ne0, + ne1, ); } - late final _ggml_sqrtPtr = _lookup< + late final _ggml_cont_2dPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sqrt'); - late final _ggml_sqrt = _ggml_sqrtPtr.asFunction< + ffi.Pointer, ffi.Int64, ffi.Int64)>>('ggml_cont_2d'); + late final _ggml_cont_2d = _ggml_cont_2dPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer, int, int)>(); - ffi.Pointer ggml_sqrt_inplace( + ffi.Pointer ggml_cont_3d( ffi.Pointer ctx, ffi.Pointer a, + int ne0, + int ne1, + int ne2, ) { - return _ggml_sqrt_inplace( + return _ggml_cont_3d( ctx, a, + ne0, + ne1, + ne2, ); } - late final _ggml_sqrt_inplacePtr = _lookup< + late final _ggml_cont_3dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sqrt_inplace'); - late final _ggml_sqrt_inplace = _ggml_sqrt_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64, + ffi.Int64)>>('ggml_cont_3d'); + late final _ggml_cont_3d = _ggml_cont_3dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int)>(); - ffi.Pointer ggml_log( + ffi.Pointer ggml_cont_4d( ffi.Pointer ctx, ffi.Pointer a, + int ne0, + int ne1, + int ne2, + int ne3, ) { - return _ggml_log( + return _ggml_cont_4d( ctx, a, + ne0, + ne1, + ne2, + ne3, ); } - late final _ggml_logPtr = _lookup< + late final _ggml_cont_4dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_log'); - late final _ggml_log = _ggml_logPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Int64)>>('ggml_cont_4d'); + late final _ggml_cont_4d = _ggml_cont_4dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); - ffi.Pointer ggml_log_inplace( + ffi.Pointer ggml_conv_1d( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int s0, + int p0, + int d0, ) { - return _ggml_log_inplace( + return _ggml_conv_1d( ctx, a, + b, + s0, + p0, + d0, ); } - late final _ggml_log_inplacePtr = _lookup< + late final _ggml_conv_1dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_log_inplace'); - late final _ggml_log_inplace = _ggml_log_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_1d'); + late final _ggml_conv_1d = _ggml_conv_1dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int, int, int)>(); - ffi.Pointer ggml_expm1( + ffi.Pointer ggml_conv_1d_dw( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int s0, + int p0, + int d0, ) { - return _ggml_expm1( + return _ggml_conv_1d_dw( ctx, a, + b, + s0, + p0, + d0, ); } - late final _ggml_expm1Ptr = _lookup< + late final _ggml_conv_1d_dwPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_expm1'); - late final _ggml_expm1 = _ggml_expm1Ptr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_1d_dw'); + late final _ggml_conv_1d_dw = _ggml_conv_1d_dwPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int, int, int)>(); - ffi.Pointer ggml_expm1_inplace( + ffi.Pointer ggml_conv_1d_dw_ph( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int s0, + int d0, ) { - return _ggml_expm1_inplace( + return _ggml_conv_1d_dw_ph( ctx, a, + b, + s0, + d0, ); } - late final _ggml_expm1_inplacePtr = _lookup< + late final _ggml_conv_1d_dw_phPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_expm1_inplace'); - late final _ggml_expm1_inplace = _ggml_expm1_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int)>>('ggml_conv_1d_dw_ph'); + late final _ggml_conv_1d_dw_ph = _ggml_conv_1d_dw_phPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int, int)>(); - ffi.Pointer ggml_softplus( + ffi.Pointer ggml_conv_1d_ph( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int s, + int d, ) { - return _ggml_softplus( + return _ggml_conv_1d_ph( ctx, a, + b, + s, + d, ); } - late final _ggml_softplusPtr = _lookup< + late final _ggml_conv_1d_phPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_softplus'); - late final _ggml_softplus = _ggml_softplusPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int)>>('ggml_conv_1d_ph'); + late final _ggml_conv_1d_ph = _ggml_conv_1d_phPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int, int)>(); - ffi.Pointer ggml_softplus_inplace( + ffi.Pointer ggml_conv_2d( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, ) { - return _ggml_softplus_inplace( + return _ggml_conv_2d( ctx, a, + b, + s0, + s1, + p0, + p1, + d0, + d1, ); } - late final _ggml_softplus_inplacePtr = _lookup< + late final _ggml_conv_2dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_softplus_inplace'); - late final _ggml_softplus_inplace = _ggml_softplus_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_2d'); + late final _ggml_conv_2d = _ggml_conv_2dPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); - - ffi.Pointer ggml_sin( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_sin( - ctx, - a, - ); - } - - late final _ggml_sinPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sin'); - late final _ggml_sin = _ggml_sinPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int, + int, + int)>(); - ffi.Pointer ggml_sin_inplace( + ffi.Pointer ggml_conv_2d_direct( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, ) { - return _ggml_sin_inplace( + return _ggml_conv_2d_direct( ctx, a, + b, + s0, + s1, + p0, + p1, + d0, + d1, ); } - late final _ggml_sin_inplacePtr = _lookup< + late final _ggml_conv_2d_directPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sin_inplace'); - late final _ggml_sin_inplace = _ggml_sin_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_2d_direct'); + late final _ggml_conv_2d_direct = _ggml_conv_2d_directPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int, + int, + int)>(); - ffi.Pointer ggml_cos( + ffi.Pointer ggml_conv_2d_dw( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1, ) { - return _ggml_cos( + return _ggml_conv_2d_dw( ctx, a, + b, + s0, + s1, + p0, + p1, + d0, + d1, ); } - late final _ggml_cosPtr = _lookup< + late final _ggml_conv_2d_dwPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_cos'); - late final _ggml_cos = _ggml_cosPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_2d_dw'); + late final _ggml_conv_2d_dw = _ggml_conv_2d_dwPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int, + int, + int)>(); - ffi.Pointer ggml_cos_inplace( + ffi.Pointer ggml_conv_2d_dw_direct( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int stride0, + int stride1, + int pad0, + int pad1, + int dilation0, + int dilation1, ) { - return _ggml_cos_inplace( + return _ggml_conv_2d_dw_direct( ctx, a, + b, + stride0, + stride1, + pad0, + pad1, + dilation0, + dilation1, ); } - late final _ggml_cos_inplacePtr = _lookup< + late final _ggml_conv_2d_dw_directPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_cos_inplace'); - late final _ggml_cos_inplace = _ggml_cos_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_2d_dw_direct'); + late final _ggml_conv_2d_dw_direct = _ggml_conv_2d_dw_directPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int, + int, + int)>(); - ffi.Pointer ggml_sum( + ffi.Pointer ggml_conv_2d_s1_ph( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_sum( + return _ggml_conv_2d_s1_ph( ctx, a, + b, ); } - late final _ggml_sumPtr = _lookup< + late final _ggml_conv_2d_s1_phPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sum'); - late final _ggml_sum = _ggml_sumPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_conv_2d_s1_ph'); + late final _ggml_conv_2d_s1_ph = _ggml_conv_2d_s1_phPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_sum_rows( + ffi.Pointer ggml_conv_2d_sk_p0( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_sum_rows( + return _ggml_conv_2d_sk_p0( ctx, a, + b, ); } - late final _ggml_sum_rowsPtr = _lookup< + late final _ggml_conv_2d_sk_p0Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sum_rows'); - late final _ggml_sum_rows = _ggml_sum_rowsPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_conv_2d_sk_p0'); + late final _ggml_conv_2d_sk_p0 = _ggml_conv_2d_sk_p0Ptr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_cumsum( + ffi.Pointer ggml_conv_3d( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int IC, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2, ) { - return _ggml_cumsum( + return _ggml_conv_3d( ctx, a, + b, + IC, + s0, + s1, + s2, + p0, + p1, + p2, + d0, + d1, + d2, ); } - late final _ggml_cumsumPtr = _lookup< + late final _ggml_conv_3dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_cumsum'); - late final _ggml_cumsum = _ggml_cumsumPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_3d'); + late final _ggml_conv_3d = _ggml_conv_3dPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int, + int, + int, + int, + int, + int, + int)>(); - ffi.Pointer ggml_mean( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_mean( - ctx, - a, - ); - } - - late final _ggml_meanPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_mean'); - late final _ggml_mean = _ggml_meanPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); - - ffi.Pointer ggml_argmax( + ffi.Pointer ggml_conv_3d_direct( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + int s0, + int s1, + int s2, + int p0, + int p1, + int p2, + int d0, + int d1, + int d2, + int n_channels, + int n_batch, + int n_channels_out, ) { - return _ggml_argmax( + return _ggml_conv_3d_direct( ctx, a, + b, + s0, + s1, + s2, + p0, + p1, + p2, + d0, + d1, + d2, + n_channels, + n_batch, + n_channels_out, ); } - late final _ggml_argmaxPtr = _lookup< + late final _ggml_conv_3d_directPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_argmax'); - late final _ggml_argmax = _ggml_argmaxPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_3d_direct'); + late final _ggml_conv_3d_direct = _ggml_conv_3d_directPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int, + int, + int, + int, + int, + int, + int, + int, + int)>(); - ffi.Pointer ggml_count_equal( + ffi.Pointer ggml_conv_transpose_1d( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, + int s0, + int p0, + int d0, ) { - return _ggml_count_equal( + return _ggml_conv_transpose_1d( ctx, a, b, + s0, + p0, + d0, ); } - late final _ggml_count_equalPtr = _lookup< + late final _ggml_conv_transpose_1dPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('ggml_count_equal'); - late final _ggml_count_equal = _ggml_count_equalPtr.asFunction< + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_conv_transpose_1d'); + late final _ggml_conv_transpose_1d = _ggml_conv_transpose_1dPtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer, int, int, int)>(); - ffi.Pointer ggml_repeat( + ffi.Pointer ggml_conv_transpose_2d_p0( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, + int stride, ) { - return _ggml_repeat( + return _ggml_conv_transpose_2d_p0( ctx, a, b, + stride, ); } - late final _ggml_repeatPtr = _lookup< + late final _ggml_conv_transpose_2d_p0Ptr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('ggml_repeat'); - late final _ggml_repeat = _ggml_repeatPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Int)>>('ggml_conv_transpose_2d_p0'); + late final _ggml_conv_transpose_2d_p0 = + _ggml_conv_transpose_2d_p0Ptr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_repeat_4d( + ffi.Pointer ggml_cos( ffi.Pointer ctx, ffi.Pointer a, - int ne0, - int ne1, - int ne2, - int ne3, ) { - return _ggml_repeat_4d( + return _ggml_cos( ctx, a, - ne0, - ne1, - ne2, - ne3, ); } - late final _ggml_repeat_4dPtr = _lookup< + late final _ggml_cosPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_repeat_4d'); - late final _ggml_repeat_4d = _ggml_repeat_4dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_cos'); + late final _ggml_cos = _ggml_cosPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_repeat_back( + ffi.Pointer ggml_cos_inplace( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_repeat_back( + return _ggml_cos_inplace( ctx, a, - b, ); } - late final _ggml_repeat_backPtr = _lookup< + late final _ggml_cos_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_repeat_back'); - late final _ggml_repeat_back = _ggml_repeat_backPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_cos_inplace'); + late final _ggml_cos_inplace = _ggml_cos_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_concat( + ffi.Pointer ggml_count_equal( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, - int dim, ) { - return _ggml_concat( + return _ggml_count_equal( ctx, a, b, - dim, ); } - late final _ggml_concatPtr = _lookup< + late final _ggml_count_equalPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Int)>>('ggml_concat'); - late final _ggml_concat = _ggml_concatPtr.asFunction< + ffi.Pointer)>>('ggml_count_equal'); + late final _ggml_count_equal = _ggml_count_equalPtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_abs( - ffi.Pointer ctx, - ffi.Pointer a, + void ggml_cpu_bf16_to_fp32( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_abs( - ctx, - a, + return _ggml_cpu_bf16_to_fp32( + arg0, + arg1, + arg2, ); } - late final _ggml_absPtr = _lookup< + late final _ggml_cpu_bf16_to_fp32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_abs'); - late final _ggml_abs = _ggml_absPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_cpu_bf16_to_fp32'); + late final _ggml_cpu_bf16_to_fp32 = _ggml_cpu_bf16_to_fp32Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_abs_inplace( - ffi.Pointer ctx, - ffi.Pointer a, + void ggml_cpu_fp16_to_fp32( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_abs_inplace( - ctx, - a, + return _ggml_cpu_fp16_to_fp32( + arg0, + arg1, + arg2, ); } - late final _ggml_abs_inplacePtr = _lookup< + late final _ggml_cpu_fp16_to_fp32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_abs_inplace'); - late final _ggml_abs_inplace = _ggml_abs_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_cpu_fp16_to_fp32'); + late final _ggml_cpu_fp16_to_fp32 = _ggml_cpu_fp16_to_fp32Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_sgn( - ffi.Pointer ctx, - ffi.Pointer a, + void ggml_cpu_fp32_to_bf16( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_sgn( - ctx, - a, + return _ggml_cpu_fp32_to_bf16( + arg0, + arg1, + arg2, ); } - late final _ggml_sgnPtr = _lookup< + late final _ggml_cpu_fp32_to_bf16Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sgn'); - late final _ggml_sgn = _ggml_sgnPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_cpu_fp32_to_bf16'); + late final _ggml_cpu_fp32_to_bf16 = _ggml_cpu_fp32_to_bf16Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_sgn_inplace( - ffi.Pointer ctx, - ffi.Pointer a, + void ggml_cpu_fp32_to_fp16( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_sgn_inplace( - ctx, - a, + return _ggml_cpu_fp32_to_fp16( + arg0, + arg1, + arg2, ); } - late final _ggml_sgn_inplacePtr = _lookup< + late final _ggml_cpu_fp32_to_fp16Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sgn_inplace'); - late final _ggml_sgn_inplace = _ggml_sgn_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_cpu_fp32_to_fp16'); + late final _ggml_cpu_fp32_to_fp16 = _ggml_cpu_fp32_to_fp16Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_neg( - ffi.Pointer ctx, - ffi.Pointer a, + void ggml_cpu_fp32_to_fp32( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_neg( - ctx, - a, + return _ggml_cpu_fp32_to_fp32( + arg0, + arg1, + arg2, ); } - late final _ggml_negPtr = _lookup< + late final _ggml_cpu_fp32_to_fp32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_neg'); - late final _ggml_neg = _ggml_negPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_cpu_fp32_to_fp32'); + late final _ggml_cpu_fp32_to_fp32 = _ggml_cpu_fp32_to_fp32Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_neg_inplace( - ffi.Pointer ctx, - ffi.Pointer a, + void ggml_cpu_fp32_to_i32( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_neg_inplace( - ctx, - a, + return _ggml_cpu_fp32_to_i32( + arg0, + arg1, + arg2, ); } - late final _ggml_neg_inplacePtr = _lookup< + late final _ggml_cpu_fp32_to_i32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_neg_inplace'); - late final _ggml_neg_inplace = _ggml_neg_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_cpu_fp32_to_i32'); + late final _ggml_cpu_fp32_to_i32 = _ggml_cpu_fp32_to_i32Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_step( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_step( - ctx, - a, - ); + int ggml_cpu_get_rvv_vlen() { + return _ggml_cpu_get_rvv_vlen(); } - late final _ggml_stepPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_step'); - late final _ggml_step = _ggml_stepPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_get_rvv_vlenPtr = + _lookup>('ggml_cpu_get_rvv_vlen'); + late final _ggml_cpu_get_rvv_vlen = + _ggml_cpu_get_rvv_vlenPtr.asFunction(); - ffi.Pointer ggml_step_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_step_inplace( - ctx, - a, - ); + int ggml_cpu_get_sve_cnt() { + return _ggml_cpu_get_sve_cnt(); } - late final _ggml_step_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_step_inplace'); - late final _ggml_step_inplace = _ggml_step_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_get_sve_cntPtr = + _lookup>('ggml_cpu_get_sve_cnt'); + late final _ggml_cpu_get_sve_cnt = + _ggml_cpu_get_sve_cntPtr.asFunction(); - ffi.Pointer ggml_tanh( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_tanh( - ctx, - a, - ); + int ggml_cpu_has_amx_int8() { + return _ggml_cpu_has_amx_int8(); } - late final _ggml_tanhPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_tanh'); - late final _ggml_tanh = _ggml_tanhPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_amx_int8Ptr = + _lookup>('ggml_cpu_has_amx_int8'); + late final _ggml_cpu_has_amx_int8 = + _ggml_cpu_has_amx_int8Ptr.asFunction(); - ffi.Pointer ggml_tanh_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_tanh_inplace( - ctx, - a, - ); + int ggml_cpu_has_arm_fma() { + return _ggml_cpu_has_arm_fma(); } - late final _ggml_tanh_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_tanh_inplace'); - late final _ggml_tanh_inplace = _ggml_tanh_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_arm_fmaPtr = + _lookup>('ggml_cpu_has_arm_fma'); + late final _ggml_cpu_has_arm_fma = + _ggml_cpu_has_arm_fmaPtr.asFunction(); - ffi.Pointer ggml_elu( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_elu( - ctx, - a, - ); + int ggml_cpu_has_avx() { + return _ggml_cpu_has_avx(); } - late final _ggml_eluPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_elu'); - late final _ggml_elu = _ggml_eluPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_avxPtr = + _lookup>('ggml_cpu_has_avx'); + late final _ggml_cpu_has_avx = + _ggml_cpu_has_avxPtr.asFunction(); - ffi.Pointer ggml_elu_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_elu_inplace( - ctx, - a, - ); + int ggml_cpu_has_avx2() { + return _ggml_cpu_has_avx2(); } - late final _ggml_elu_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_elu_inplace'); - late final _ggml_elu_inplace = _ggml_elu_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_avx2Ptr = + _lookup>('ggml_cpu_has_avx2'); + late final _ggml_cpu_has_avx2 = + _ggml_cpu_has_avx2Ptr.asFunction(); - ffi.Pointer ggml_relu( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_relu( - ctx, - a, - ); + int ggml_cpu_has_avx512() { + return _ggml_cpu_has_avx512(); } - late final _ggml_reluPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_relu'); - late final _ggml_relu = _ggml_reluPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_avx512Ptr = + _lookup>('ggml_cpu_has_avx512'); + late final _ggml_cpu_has_avx512 = + _ggml_cpu_has_avx512Ptr.asFunction(); - ffi.Pointer ggml_leaky_relu( - ffi.Pointer ctx, - ffi.Pointer a, - double negative_slope, - bool inplace, - ) { - return _ggml_leaky_relu( - ctx, - a, - negative_slope, - inplace, - ); + int ggml_cpu_has_avx512_bf16() { + return _ggml_cpu_has_avx512_bf16(); } - late final _ggml_leaky_reluPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Bool)>>('ggml_leaky_relu'); - late final _ggml_leaky_relu = _ggml_leaky_reluPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double, bool)>(); - - ffi.Pointer ggml_relu_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_relu_inplace( - ctx, - a, - ); + late final _ggml_cpu_has_avx512_bf16Ptr = + _lookup>( + 'ggml_cpu_has_avx512_bf16'); + late final _ggml_cpu_has_avx512_bf16 = + _ggml_cpu_has_avx512_bf16Ptr.asFunction(); + + int ggml_cpu_has_avx512_vbmi() { + return _ggml_cpu_has_avx512_vbmi(); } - late final _ggml_relu_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_relu_inplace'); - late final _ggml_relu_inplace = _ggml_relu_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_avx512_vbmiPtr = + _lookup>( + 'ggml_cpu_has_avx512_vbmi'); + late final _ggml_cpu_has_avx512_vbmi = + _ggml_cpu_has_avx512_vbmiPtr.asFunction(); - ffi.Pointer ggml_sigmoid( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_sigmoid( - ctx, - a, - ); + int ggml_cpu_has_avx512_vnni() { + return _ggml_cpu_has_avx512_vnni(); } - late final _ggml_sigmoidPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sigmoid'); - late final _ggml_sigmoid = _ggml_sigmoidPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_avx512_vnniPtr = + _lookup>( + 'ggml_cpu_has_avx512_vnni'); + late final _ggml_cpu_has_avx512_vnni = + _ggml_cpu_has_avx512_vnniPtr.asFunction(); - ffi.Pointer ggml_sigmoid_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_sigmoid_inplace( - ctx, - a, - ); + int ggml_cpu_has_avx_vnni() { + return _ggml_cpu_has_avx_vnni(); } - late final _ggml_sigmoid_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_sigmoid_inplace'); - late final _ggml_sigmoid_inplace = _ggml_sigmoid_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_avx_vnniPtr = + _lookup>('ggml_cpu_has_avx_vnni'); + late final _ggml_cpu_has_avx_vnni = + _ggml_cpu_has_avx_vnniPtr.asFunction(); - ffi.Pointer ggml_gelu( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_gelu( - ctx, - a, - ); + int ggml_cpu_has_bmi2() { + return _ggml_cpu_has_bmi2(); } - late final _ggml_geluPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_gelu'); - late final _ggml_gelu = _ggml_geluPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_bmi2Ptr = + _lookup>('ggml_cpu_has_bmi2'); + late final _ggml_cpu_has_bmi2 = + _ggml_cpu_has_bmi2Ptr.asFunction(); - ffi.Pointer ggml_gelu_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_gelu_inplace( - ctx, - a, - ); + int ggml_cpu_has_dotprod() { + return _ggml_cpu_has_dotprod(); } - late final _ggml_gelu_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_gelu_inplace'); - late final _ggml_gelu_inplace = _ggml_gelu_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_dotprodPtr = + _lookup>('ggml_cpu_has_dotprod'); + late final _ggml_cpu_has_dotprod = + _ggml_cpu_has_dotprodPtr.asFunction(); - ffi.Pointer ggml_gelu_erf( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_gelu_erf( - ctx, - a, - ); + int ggml_cpu_has_f16c() { + return _ggml_cpu_has_f16c(); } - late final _ggml_gelu_erfPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_gelu_erf'); - late final _ggml_gelu_erf = _ggml_gelu_erfPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_f16cPtr = + _lookup>('ggml_cpu_has_f16c'); + late final _ggml_cpu_has_f16c = + _ggml_cpu_has_f16cPtr.asFunction(); - ffi.Pointer ggml_gelu_erf_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_gelu_erf_inplace( - ctx, - a, - ); + int ggml_cpu_has_fma() { + return _ggml_cpu_has_fma(); } - late final _ggml_gelu_erf_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_gelu_erf_inplace'); - late final _ggml_gelu_erf_inplace = _ggml_gelu_erf_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_fmaPtr = + _lookup>('ggml_cpu_has_fma'); + late final _ggml_cpu_has_fma = + _ggml_cpu_has_fmaPtr.asFunction(); - ffi.Pointer ggml_gelu_quick( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_gelu_quick( - ctx, - a, - ); + int ggml_cpu_has_fp16_va() { + return _ggml_cpu_has_fp16_va(); } - late final _ggml_gelu_quickPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_gelu_quick'); - late final _ggml_gelu_quick = _ggml_gelu_quickPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_fp16_vaPtr = + _lookup>('ggml_cpu_has_fp16_va'); + late final _ggml_cpu_has_fp16_va = + _ggml_cpu_has_fp16_vaPtr.asFunction(); - ffi.Pointer ggml_gelu_quick_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_gelu_quick_inplace( - ctx, - a, - ); + int ggml_cpu_has_llamafile() { + return _ggml_cpu_has_llamafile(); } - late final _ggml_gelu_quick_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_gelu_quick_inplace'); - late final _ggml_gelu_quick_inplace = _ggml_gelu_quick_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_llamafilePtr = + _lookup>('ggml_cpu_has_llamafile'); + late final _ggml_cpu_has_llamafile = + _ggml_cpu_has_llamafilePtr.asFunction(); - ffi.Pointer ggml_silu( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_silu( - ctx, - a, - ); + int ggml_cpu_has_matmul_int8() { + return _ggml_cpu_has_matmul_int8(); } - late final _ggml_siluPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_silu'); - late final _ggml_silu = _ggml_siluPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_matmul_int8Ptr = + _lookup>( + 'ggml_cpu_has_matmul_int8'); + late final _ggml_cpu_has_matmul_int8 = + _ggml_cpu_has_matmul_int8Ptr.asFunction(); - ffi.Pointer ggml_silu_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_silu_inplace( - ctx, - a, - ); + int ggml_cpu_has_neon() { + return _ggml_cpu_has_neon(); } - late final _ggml_silu_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_silu_inplace'); - late final _ggml_silu_inplace = _ggml_silu_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_neonPtr = + _lookup>('ggml_cpu_has_neon'); + late final _ggml_cpu_has_neon = + _ggml_cpu_has_neonPtr.asFunction(); - ffi.Pointer ggml_silu_back( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ) { - return _ggml_silu_back( - ctx, - a, - b, - ); + int ggml_cpu_has_riscv_v() { + return _ggml_cpu_has_riscv_v(); } - late final _ggml_silu_backPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_silu_back'); - late final _ggml_silu_back = _ggml_silu_backPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_riscv_vPtr = + _lookup>('ggml_cpu_has_riscv_v'); + late final _ggml_cpu_has_riscv_v = + _ggml_cpu_has_riscv_vPtr.asFunction(); - ffi.Pointer ggml_hardswish( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_hardswish( - ctx, - a, - ); + int ggml_cpu_has_sme() { + return _ggml_cpu_has_sme(); } - late final _ggml_hardswishPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_hardswish'); - late final _ggml_hardswish = _ggml_hardswishPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_smePtr = + _lookup>('ggml_cpu_has_sme'); + late final _ggml_cpu_has_sme = + _ggml_cpu_has_smePtr.asFunction(); - ffi.Pointer ggml_hardsigmoid( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_hardsigmoid( - ctx, - a, - ); + int ggml_cpu_has_sme2() { + return _ggml_cpu_has_sme2(); } - late final _ggml_hardsigmoidPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_hardsigmoid'); - late final _ggml_hardsigmoid = _ggml_hardsigmoidPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_sme2Ptr = + _lookup>('ggml_cpu_has_sme2'); + late final _ggml_cpu_has_sme2 = + _ggml_cpu_has_sme2Ptr.asFunction(); - ffi.Pointer ggml_exp( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_exp( - ctx, - a, - ); + int ggml_cpu_has_sse3() { + return _ggml_cpu_has_sse3(); } - late final _ggml_expPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_exp'); - late final _ggml_exp = _ggml_expPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_sse3Ptr = + _lookup>('ggml_cpu_has_sse3'); + late final _ggml_cpu_has_sse3 = + _ggml_cpu_has_sse3Ptr.asFunction(); - ffi.Pointer ggml_exp_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_exp_inplace( - ctx, - a, - ); + int ggml_cpu_has_ssse3() { + return _ggml_cpu_has_ssse3(); } - late final _ggml_exp_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_exp_inplace'); - late final _ggml_exp_inplace = _ggml_exp_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_ssse3Ptr = + _lookup>('ggml_cpu_has_ssse3'); + late final _ggml_cpu_has_ssse3 = + _ggml_cpu_has_ssse3Ptr.asFunction(); - ffi.Pointer ggml_floor( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_floor( - ctx, - a, - ); + int ggml_cpu_has_sve() { + return _ggml_cpu_has_sve(); } - late final _ggml_floorPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_floor'); - late final _ggml_floor = _ggml_floorPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_svePtr = + _lookup>('ggml_cpu_has_sve'); + late final _ggml_cpu_has_sve = + _ggml_cpu_has_svePtr.asFunction(); - ffi.Pointer ggml_floor_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_floor_inplace( - ctx, - a, - ); + int ggml_cpu_has_vsx() { + return _ggml_cpu_has_vsx(); } - late final _ggml_floor_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_floor_inplace'); - late final _ggml_floor_inplace = _ggml_floor_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_vsxPtr = + _lookup>('ggml_cpu_has_vsx'); + late final _ggml_cpu_has_vsx = + _ggml_cpu_has_vsxPtr.asFunction(); - ffi.Pointer ggml_ceil( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_ceil( - ctx, - a, - ); + int ggml_cpu_has_vxe() { + return _ggml_cpu_has_vxe(); } - late final _ggml_ceilPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_ceil'); - late final _ggml_ceil = _ggml_ceilPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_vxePtr = + _lookup>('ggml_cpu_has_vxe'); + late final _ggml_cpu_has_vxe = + _ggml_cpu_has_vxePtr.asFunction(); - ffi.Pointer ggml_ceil_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_ceil_inplace( - ctx, - a, - ); + int ggml_cpu_has_wasm_simd() { + return _ggml_cpu_has_wasm_simd(); } - late final _ggml_ceil_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_ceil_inplace'); - late final _ggml_ceil_inplace = _ggml_ceil_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cpu_has_wasm_simdPtr = + _lookup>('ggml_cpu_has_wasm_simd'); + late final _ggml_cpu_has_wasm_simd = + _ggml_cpu_has_wasm_simdPtr.asFunction(); - ffi.Pointer ggml_round( + void ggml_cpu_init() { + return _ggml_cpu_init(); + } + + late final _ggml_cpu_initPtr = + _lookup>('ggml_cpu_init'); + late final _ggml_cpu_init = _ggml_cpu_initPtr.asFunction(); + + ffi.Pointer ggml_cpy( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_round( + return _ggml_cpy( ctx, a, + b, ); } - late final _ggml_roundPtr = _lookup< + late final _ggml_cpyPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_round'); - late final _ggml_round = _ggml_roundPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer)>>('ggml_cpy'); + late final _ggml_cpy = _ggml_cpyPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_round_inplace( + ffi.Pointer ggml_cross_entropy_loss( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_round_inplace( + return _ggml_cross_entropy_loss( ctx, a, + b, ); } - late final _ggml_round_inplacePtr = _lookup< + late final _ggml_cross_entropy_lossPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_round_inplace'); - late final _ggml_round_inplace = _ggml_round_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_cross_entropy_loss'); + late final _ggml_cross_entropy_loss = _ggml_cross_entropy_lossPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - /// Truncates the fractional part of each element in the tensor (towards zero). - /// For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0 - /// Similar to std::trunc in C/C++. - ffi.Pointer ggml_trunc( + ffi.Pointer ggml_cross_entropy_loss_back( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, ) { - return _ggml_trunc( + return _ggml_cross_entropy_loss_back( ctx, a, + b, + c, ); } - late final _ggml_truncPtr = _lookup< + late final _ggml_cross_entropy_loss_backPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_trunc'); - late final _ggml_trunc = _ggml_truncPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_cross_entropy_loss_back'); + late final _ggml_cross_entropy_loss_back = + _ggml_cross_entropy_loss_backPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - ffi.Pointer ggml_trunc_inplace( + ffi.Pointer ggml_cumsum( ffi.Pointer ctx, ffi.Pointer a, ) { - return _ggml_trunc_inplace( + return _ggml_cumsum( ctx, a, ); } - late final _ggml_trunc_inplacePtr = _lookup< + late final _ggml_cumsumPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_trunc_inplace'); - late final _ggml_trunc_inplace = _ggml_trunc_inplacePtr.asFunction< + ffi.Pointer)>>('ggml_cumsum'); + late final _ggml_cumsum = _ggml_cumsumPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_xielu( + ffi.Pointer ggml_custom_4d( ffi.Pointer ctx, - ffi.Pointer a, - double alpha_n, - double alpha_p, - double beta, - double eps, + ggml_type type, + int ne0, + int ne1, + int ne2, + int ne3, + ffi.Pointer> args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + ffi.Pointer userdata, ) { - return _ggml_xielu( + return _ggml_custom_4d( ctx, - a, - alpha_n, - alpha_p, - beta, - eps, + type.value, + ne0, + ne1, + ne2, + ne3, + args, + n_args, + fun, + n_tasks, + userdata, ); } - late final _ggml_xieluPtr = _lookup< + late final _ggml_custom_4dPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_xielu'); - late final _ggml_xielu = _ggml_xieluPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, double, double, double, double)>(); + ffi.UnsignedInt, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Pointer>, + ffi.Int, + ggml_custom_op_t, + ffi.Int, + ffi.Pointer)>>('ggml_custom_4d'); + late final _ggml_custom_4d = _ggml_custom_4dPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + int, + int, + int, + int, + int, + ffi.Pointer>, + int, + ggml_custom_op_t, + int, + ffi.Pointer)>(); - ffi.Pointer ggml_glu( + ffi.Pointer ggml_custom_inplace( ffi.Pointer ctx, ffi.Pointer a, - ggml_glu_op op, - bool swapped, + ffi.Pointer> args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + ffi.Pointer userdata, ) { - return _ggml_glu( + return _ggml_custom_inplace( ctx, a, - op.value, - swapped, + args, + n_args, + fun, + n_tasks, + userdata, ); } - late final _ggml_gluPtr = _lookup< + late final _ggml_custom_inplacePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.UnsignedInt, - ffi.Bool)>>('ggml_glu'); - late final _ggml_glu = _ggml_gluPtr.asFunction< + ffi.Pointer>, + ffi.Int, + ggml_custom_op_t, + ffi.Int, + ffi.Pointer)>>('ggml_custom_inplace'); + late final _ggml_custom_inplace = _ggml_custom_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, bool)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>, + int, + ggml_custom_op_t, + int, + ffi.Pointer)>(); - ffi.Pointer ggml_reglu( - ffi.Pointer ctx, - ffi.Pointer a, - ) { - return _ggml_reglu( - ctx, - a, - ); + int ggml_cycles() { + return _ggml_cycles(); } - late final _ggml_regluPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_reglu'); - late final _ggml_reglu = _ggml_regluPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_cyclesPtr = + _lookup>('ggml_cycles'); + late final _ggml_cycles = _ggml_cyclesPtr.asFunction(); - ffi.Pointer ggml_reglu_swapped( + int ggml_cycles_per_ms() { + return _ggml_cycles_per_ms(); + } + + late final _ggml_cycles_per_msPtr = + _lookup>('ggml_cycles_per_ms'); + late final _ggml_cycles_per_ms = + _ggml_cycles_per_msPtr.asFunction(); + + ffi.Pointer ggml_diag( ffi.Pointer ctx, ffi.Pointer a, ) { - return _ggml_reglu_swapped( + return _ggml_diag( ctx, a, ); } - late final _ggml_reglu_swappedPtr = _lookup< + late final _ggml_diagPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_reglu_swapped'); - late final _ggml_reglu_swapped = _ggml_reglu_swappedPtr.asFunction< + ffi.Pointer)>>('ggml_diag'); + late final _ggml_diag = _ggml_diagPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_geglu( + ffi.Pointer ggml_diag_mask_inf( ffi.Pointer ctx, ffi.Pointer a, + int n_past, ) { - return _ggml_geglu( + return _ggml_diag_mask_inf( ctx, a, + n_past, ); } - late final _ggml_gegluPtr = _lookup< + late final _ggml_diag_mask_infPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_geglu'); - late final _ggml_geglu = _ggml_gegluPtr.asFunction< + ffi.Pointer, ffi.Int)>>('ggml_diag_mask_inf'); + late final _ggml_diag_mask_inf = _ggml_diag_mask_infPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_geglu_swapped( + ffi.Pointer ggml_diag_mask_inf_inplace( ffi.Pointer ctx, ffi.Pointer a, + int n_past, ) { - return _ggml_geglu_swapped( + return _ggml_diag_mask_inf_inplace( ctx, a, + n_past, ); } - late final _ggml_geglu_swappedPtr = _lookup< + late final _ggml_diag_mask_inf_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_geglu_swapped'); - late final _ggml_geglu_swapped = _ggml_geglu_swappedPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int)>>('ggml_diag_mask_inf_inplace'); + late final _ggml_diag_mask_inf_inplace = + _ggml_diag_mask_inf_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_swiglu( + ffi.Pointer ggml_diag_mask_zero( ffi.Pointer ctx, ffi.Pointer a, + int n_past, ) { - return _ggml_swiglu( + return _ggml_diag_mask_zero( ctx, a, + n_past, ); } - late final _ggml_swigluPtr = _lookup< + late final _ggml_diag_mask_zeroPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_swiglu'); - late final _ggml_swiglu = _ggml_swigluPtr.asFunction< + ffi.Pointer, ffi.Int)>>('ggml_diag_mask_zero'); + late final _ggml_diag_mask_zero = _ggml_diag_mask_zeroPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_swiglu_swapped( + ffi.Pointer ggml_diag_mask_zero_inplace( ffi.Pointer ctx, ffi.Pointer a, + int n_past, ) { - return _ggml_swiglu_swapped( + return _ggml_diag_mask_zero_inplace( ctx, a, + n_past, ); } - late final _ggml_swiglu_swappedPtr = _lookup< + late final _ggml_diag_mask_zero_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_swiglu_swapped'); - late final _ggml_swiglu_swapped = _ggml_swiglu_swappedPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int)>>('ggml_diag_mask_zero_inplace'); + late final _ggml_diag_mask_zero_inplace = + _ggml_diag_mask_zero_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_geglu_erf( + ffi.Pointer ggml_div( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_geglu_erf( + return _ggml_div( ctx, a, + b, ); } - late final _ggml_geglu_erfPtr = _lookup< + late final _ggml_divPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_geglu_erf'); - late final _ggml_geglu_erf = _ggml_geglu_erfPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer)>>('ggml_div'); + late final _ggml_div = _ggml_divPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_geglu_erf_swapped( + ffi.Pointer ggml_div_inplace( ffi.Pointer ctx, ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_geglu_erf_swapped( + return _ggml_div_inplace( ctx, a, + b, ); } - late final _ggml_geglu_erf_swappedPtr = _lookup< + late final _ggml_div_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_geglu_erf_swapped'); - late final _ggml_geglu_erf_swapped = _ggml_geglu_erf_swappedPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_div_inplace'); + late final _ggml_div_inplace = _ggml_div_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_geglu_quick( + ffi.Pointer ggml_dsv4_hc_comb( ffi.Pointer ctx, - ffi.Pointer a, + ffi.Pointer mixes, + ffi.Pointer scale, + ffi.Pointer base, + double eps, + int n_iter, ) { - return _ggml_geglu_quick( + return _ggml_dsv4_hc_comb( ctx, - a, + mixes, + scale, + base, + eps, + n_iter, ); } - late final _ggml_geglu_quickPtr = _lookup< + late final _ggml_dsv4_hc_combPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_geglu_quick'); - late final _ggml_geglu_quick = _ggml_geglu_quickPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Int32)>>('ggml_dsv4_hc_comb'); + late final _ggml_dsv4_hc_comb = _ggml_dsv4_hc_combPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + double, + int)>(); - ffi.Pointer ggml_geglu_quick_swapped( + ffi.Pointer ggml_dsv4_hc_post( ffi.Pointer ctx, - ffi.Pointer a, + ffi.Pointer x, + ffi.Pointer residual, + ffi.Pointer post, + ffi.Pointer comb, ) { - return _ggml_geglu_quick_swapped( + return _ggml_dsv4_hc_post( ctx, - a, + x, + residual, + post, + comb, ); } - late final _ggml_geglu_quick_swappedPtr = _lookup< + late final _ggml_dsv4_hc_postPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_geglu_quick_swapped'); - late final _ggml_geglu_quick_swapped = - _ggml_geglu_quick_swappedPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_dsv4_hc_post'); + late final _ggml_dsv4_hc_post = _ggml_dsv4_hc_postPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - ffi.Pointer ggml_glu_split( + ffi.Pointer ggml_dsv4_hc_pre( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ggml_glu_op op, + ffi.Pointer x, + ffi.Pointer weights, ) { - return _ggml_glu_split( + return _ggml_dsv4_hc_pre( ctx, - a, - b, - op.value, + x, + weights, ); } - late final _ggml_glu_splitPtr = _lookup< + late final _ggml_dsv4_hc_prePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.UnsignedInt)>>('ggml_glu_split'); - late final _ggml_glu_split = _ggml_glu_splitPtr.asFunction< + ffi.Pointer)>>('ggml_dsv4_hc_pre'); + late final _ggml_dsv4_hc_pre = _ggml_dsv4_hc_prePtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_reglu_split( + ffi.Pointer ggml_dup( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_reglu_split( + return _ggml_dup( ctx, a, - b, ); } - late final _ggml_reglu_splitPtr = _lookup< + late final _ggml_dupPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_reglu_split'); - late final _ggml_reglu_split = _ggml_reglu_splitPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_dup'); + late final _ggml_dup = _ggml_dupPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_geglu_split( + ffi.Pointer ggml_dup_inplace( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_geglu_split( + return _ggml_dup_inplace( ctx, a, - b, ); } - late final _ggml_geglu_splitPtr = _lookup< + late final _ggml_dup_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_geglu_split'); - late final _ggml_geglu_split = _ggml_geglu_splitPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_dup_inplace'); + late final _ggml_dup_inplace = _ggml_dup_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_swiglu_split( + ffi.Pointer ggml_dup_tensor( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + ffi.Pointer src, ) { - return _ggml_swiglu_split( + return _ggml_dup_tensor( ctx, - a, - b, + src, ); } - late final _ggml_swiglu_splitPtr = _lookup< + late final _ggml_dup_tensorPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_swiglu_split'); - late final _ggml_swiglu_split = _ggml_swiglu_splitPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_dup_tensor'); + late final _ggml_dup_tensor = _ggml_dup_tensorPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_geglu_erf_split( + int ggml_element_size( + ffi.Pointer tensor, + ) { + return _ggml_element_size( + tensor, + ); + } + + late final _ggml_element_sizePtr = + _lookup)>>( + 'ggml_element_size'); + late final _ggml_element_size = _ggml_element_sizePtr + .asFunction)>(); + + ffi.Pointer ggml_elu( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_geglu_erf_split( + return _ggml_elu( ctx, a, - b, ); } - late final _ggml_geglu_erf_splitPtr = _lookup< + late final _ggml_eluPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_geglu_erf_split'); - late final _ggml_geglu_erf_split = _ggml_geglu_erf_splitPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_elu'); + late final _ggml_elu = _ggml_eluPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_geglu_quick_split( + ffi.Pointer ggml_elu_inplace( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_geglu_quick_split( + return _ggml_elu_inplace( ctx, a, - b, ); } - late final _ggml_geglu_quick_splitPtr = _lookup< + late final _ggml_elu_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_geglu_quick_split'); - late final _ggml_geglu_quick_split = _ggml_geglu_quick_splitPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_elu_inplace'); + late final _ggml_elu_inplace = _ggml_elu_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_swiglu_oai( + ffi.Pointer ggml_exp( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, - double alpha, - double limit, ) { - return _ggml_swiglu_oai( + return _ggml_exp( ctx, a, - b, - alpha, - limit, ); } - late final _ggml_swiglu_oaiPtr = _lookup< + late final _ggml_expPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float)>>('ggml_swiglu_oai'); - late final _ggml_swiglu_oai = _ggml_swiglu_oaiPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_exp'); + late final _ggml_exp = _ggml_expPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - double, - double)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_norm( + ffi.Pointer ggml_exp_inplace( ffi.Pointer ctx, ffi.Pointer a, - double eps, ) { - return _ggml_norm( + return _ggml_exp_inplace( ctx, a, - eps, ); } - late final _ggml_normPtr = _lookup< + late final _ggml_exp_inplacePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_norm'); - late final _ggml_norm = _ggml_normPtr.asFunction< + ffi.Pointer)>>('ggml_exp_inplace'); + late final _ggml_exp_inplace = _ggml_exp_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_norm_inplace( + ffi.Pointer ggml_expm1( ffi.Pointer ctx, ffi.Pointer a, - double eps, ) { - return _ggml_norm_inplace( + return _ggml_expm1( ctx, a, - eps, ); } - late final _ggml_norm_inplacePtr = _lookup< + late final _ggml_expm1Ptr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_norm_inplace'); - late final _ggml_norm_inplace = _ggml_norm_inplacePtr.asFunction< + ffi.Pointer)>>('ggml_expm1'); + late final _ggml_expm1 = _ggml_expm1Ptr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_rms_norm( + ffi.Pointer ggml_expm1_inplace( ffi.Pointer ctx, ffi.Pointer a, - double eps, ) { - return _ggml_rms_norm( + return _ggml_expm1_inplace( ctx, a, - eps, ); } - late final _ggml_rms_normPtr = _lookup< + late final _ggml_expm1_inplacePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_rms_norm'); - late final _ggml_rms_norm = _ggml_rms_normPtr.asFunction< + ffi.Pointer)>>('ggml_expm1_inplace'); + late final _ggml_expm1_inplace = _ggml_expm1_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_rms_norm_inplace( + ffi.Pointer ggml_fill( ffi.Pointer ctx, ffi.Pointer a, - double eps, + double c, ) { - return _ggml_rms_norm_inplace( + return _ggml_fill( ctx, a, - eps, + c, ); } - late final _ggml_rms_norm_inplacePtr = _lookup< + late final _ggml_fillPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_rms_norm_inplace'); - late final _ggml_rms_norm_inplace = _ggml_rms_norm_inplacePtr.asFunction< + ffi.Pointer, ffi.Float)>>('ggml_fill'); + late final _ggml_fill = _ggml_fillPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, double)>(); - ffi.Pointer ggml_group_norm( + ffi.Pointer ggml_fill_inplace( ffi.Pointer ctx, ffi.Pointer a, - int n_groups, - double eps, + double c, ) { - return _ggml_group_norm( + return _ggml_fill_inplace( ctx, a, - n_groups, - eps, + c, ); } - late final _ggml_group_normPtr = _lookup< + late final _ggml_fill_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Float)>>('ggml_group_norm'); - late final _ggml_group_norm = _ggml_group_normPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_fill_inplace'); + late final _ggml_fill_inplace = _ggml_fill_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, double)>(); + ffi.Pointer, ffi.Pointer, double)>(); - ffi.Pointer ggml_group_norm_inplace( + ffi.Pointer ggml_flash_attn_back( ffi.Pointer ctx, - ffi.Pointer a, - int n_groups, - double eps, + ffi.Pointer q, + ffi.Pointer k, + ffi.Pointer v, + ffi.Pointer d, + bool masked, ) { - return _ggml_group_norm_inplace( + return _ggml_flash_attn_back( ctx, - a, - n_groups, - eps, + q, + k, + v, + d, + masked, ); } - late final _ggml_group_norm_inplacePtr = _lookup< + late final _ggml_flash_attn_backPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Int, - ffi.Float)>>('ggml_group_norm_inplace'); - late final _ggml_group_norm_inplace = _ggml_group_norm_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, double)>(); - - ffi.Pointer ggml_l2_norm( - ffi.Pointer ctx, - ffi.Pointer a, - double eps, - ) { - return _ggml_l2_norm( - ctx, - a, - eps, - ); - } - - late final _ggml_l2_normPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_l2_norm'); - late final _ggml_l2_norm = _ggml_l2_normPtr.asFunction< + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Bool)>>('ggml_flash_attn_back'); + late final _ggml_flash_attn_back = _ggml_flash_attn_backPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + bool)>(); - ffi.Pointer ggml_l2_norm_inplace( + ffi.Pointer ggml_flash_attn_ext( ffi.Pointer ctx, - ffi.Pointer a, - double eps, + ffi.Pointer q, + ffi.Pointer k, + ffi.Pointer v, + ffi.Pointer mask, + double scale, + double max_bias, + double logit_softcap, ) { - return _ggml_l2_norm_inplace( + return _ggml_flash_attn_ext( ctx, - a, - eps, + q, + k, + v, + mask, + scale, + max_bias, + logit_softcap, ); } - late final _ggml_l2_norm_inplacePtr = _lookup< + late final _ggml_flash_attn_extPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_l2_norm_inplace'); - late final _ggml_l2_norm_inplace = _ggml_l2_norm_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_flash_attn_ext'); + late final _ggml_flash_attn_ext = _ggml_flash_attn_extPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + double, + double, + double)>(); - ffi.Pointer ggml_rms_norm_back( - ffi.Pointer ctx, + void ggml_flash_attn_ext_add_sinks( ffi.Pointer a, - ffi.Pointer b, - double eps, + ffi.Pointer sinks, ) { - return _ggml_rms_norm_back( - ctx, + return _ggml_flash_attn_ext_add_sinks( a, - b, - eps, + sinks, ); } - late final _ggml_rms_norm_backPtr = _lookup< + late final _ggml_flash_attn_ext_add_sinksPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Float)>>('ggml_rms_norm_back'); - late final _ggml_rms_norm_back = _ggml_rms_norm_backPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('ggml_flash_attn_ext_add_sinks'); + late final _ggml_flash_attn_ext_add_sinks = + _ggml_flash_attn_ext_add_sinksPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_mul_mat( - ffi.Pointer ctx, + ggml_prec ggml_flash_attn_ext_get_prec( ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_mul_mat( - ctx, + return ggml_prec.fromValue(_ggml_flash_attn_ext_get_prec( a, - b, - ); + )); } - late final _ggml_mul_matPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_mul_mat'); - late final _ggml_mul_mat = _ggml_mul_matPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_flash_attn_ext_get_precPtr = _lookup< + ffi + .NativeFunction)>>( + 'ggml_flash_attn_ext_get_prec'); + late final _ggml_flash_attn_ext_get_prec = _ggml_flash_attn_ext_get_precPtr + .asFunction)>(); - void ggml_mul_mat_set_prec( + void ggml_flash_attn_ext_set_prec( ffi.Pointer a, ggml_prec prec, ) { - return _ggml_mul_mat_set_prec( + return _ggml_flash_attn_ext_set_prec( a, prec.value, ); } - late final _ggml_mul_mat_set_precPtr = _lookup< + late final _ggml_flash_attn_ext_set_precPtr = _lookup< ffi.NativeFunction< ffi.Void Function(ffi.Pointer, - ffi.UnsignedInt)>>('ggml_mul_mat_set_prec'); - late final _ggml_mul_mat_set_prec = _ggml_mul_mat_set_precPtr + ffi.UnsignedInt)>>('ggml_flash_attn_ext_set_prec'); + late final _ggml_flash_attn_ext_set_prec = _ggml_flash_attn_ext_set_precPtr .asFunction, int)>(); - void ggml_mul_mat_set_hint( + ffi.Pointer ggml_floor( + ffi.Pointer ctx, ffi.Pointer a, - ggml_op_hint hint, ) { - return _ggml_mul_mat_set_hint( + return _ggml_floor( + ctx, a, - hint.value, ); } - late final _ggml_mul_mat_set_hintPtr = _lookup< + late final _ggml_floorPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.UnsignedInt)>>('ggml_mul_mat_set_hint'); - late final _ggml_mul_mat_set_hint = _ggml_mul_mat_set_hintPtr - .asFunction, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_floor'); + late final _ggml_floor = _ggml_floorPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_mul_mat_id( + ffi.Pointer ggml_floor_inplace( ffi.Pointer ctx, - ffi.Pointer as, - ffi.Pointer b, - ffi.Pointer ids, + ffi.Pointer a, ) { - return _ggml_mul_mat_id( + return _ggml_floor_inplace( ctx, - as, - b, - ids, + a, ); } - late final _ggml_mul_mat_idPtr = _lookup< + late final _ggml_floor_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_mul_mat_id'); - late final _ggml_mul_mat_id = _ggml_mul_mat_idPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_floor_inplace'); + late final _ggml_floor_inplace = _ggml_floor_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_out_prod( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + ffi.Pointer ggml_fopen( + ffi.Pointer fname, + ffi.Pointer mode, ) { - return _ggml_out_prod( - ctx, - a, - b, + return _ggml_fopen( + fname, + mode, ); } - late final _ggml_out_prodPtr = _lookup< + late final _ggml_fopenPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_out_prod'); - late final _ggml_out_prod = _ggml_out_prodPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>>('ggml_fopen'); + late final _ggml_fopen = _ggml_fopenPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_scale( - ffi.Pointer ctx, - ffi.Pointer a, - double s, + ffi.Pointer ggml_format_name( + ffi.Pointer tensor, + ffi.Pointer fmt, ) { - return _ggml_scale( - ctx, - a, - s, + return _ggml_format_name( + tensor, + fmt, ); } - late final _ggml_scalePtr = _lookup< + late final _ggml_format_namePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_scale'); - late final _ggml_scale = _ggml_scalePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_format_name'); + late final _ggml_format_name = _ggml_format_namePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_scale_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - double s, + double ggml_fp16_to_fp32( + int arg0, ) { - return _ggml_scale_inplace( - ctx, - a, - s, + return _ggml_fp16_to_fp32( + arg0, ); } - late final _ggml_scale_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_scale_inplace'); - late final _ggml_scale_inplace = _ggml_scale_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + late final _ggml_fp16_to_fp32Ptr = + _lookup>( + 'ggml_fp16_to_fp32'); + late final _ggml_fp16_to_fp32 = + _ggml_fp16_to_fp32Ptr.asFunction(); - ffi.Pointer ggml_scale_bias( - ffi.Pointer ctx, - ffi.Pointer a, - double s, - double b, + void ggml_fp16_to_fp32_row( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_scale_bias( - ctx, - a, - s, - b, + return _ggml_fp16_to_fp32_row( + arg0, + arg1, + arg2, ); } - late final _ggml_scale_biasPtr = _lookup< + late final _ggml_fp16_to_fp32_rowPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float)>>('ggml_scale_bias'); - late final _ggml_scale_bias = _ggml_scale_biasPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, double, double)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_fp16_to_fp32_row'); + late final _ggml_fp16_to_fp32_row = _ggml_fp16_to_fp32_rowPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_scale_bias_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - double s, - double b, + ggml_bf16_t ggml_fp32_to_bf16( + double arg0, ) { - return _ggml_scale_bias_inplace( - ctx, - a, - s, - b, + return _ggml_fp32_to_bf16( + arg0, ); } - late final _ggml_scale_bias_inplacePtr = _lookup< + late final _ggml_fp32_to_bf16Ptr = + _lookup>( + 'ggml_fp32_to_bf16'); + late final _ggml_fp32_to_bf16 = + _ggml_fp32_to_bf16Ptr.asFunction(); + + void ggml_fp32_to_bf16_row( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, + ) { + return _ggml_fp32_to_bf16_row( + arg0, + arg1, + arg2, + ); + } + + late final _ggml_fp32_to_bf16_rowPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float)>>('ggml_scale_bias_inplace'); - late final _ggml_scale_bias_inplace = _ggml_scale_bias_inplacePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, double, double)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_fp32_to_bf16_row'); + late final _ggml_fp32_to_bf16_row = _ggml_fp32_to_bf16_rowPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_set( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int nb1, - int nb2, - int nb3, - int offset, + void ggml_fp32_to_bf16_row_ref( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _ggml_set( - ctx, - a, - b, - nb1, - nb2, - nb3, - offset, + return _ggml_fp32_to_bf16_row_ref( + arg0, + arg1, + arg2, ); } - late final _ggml_setPtr = _lookup< + late final _ggml_fp32_to_bf16_row_refPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_set'); - late final _ggml_set = _ggml_setPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_fp32_to_bf16_row_ref'); + late final _ggml_fp32_to_bf16_row_ref = + _ggml_fp32_to_bf16_row_refPtr.asFunction< + void Function( + ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_set_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int nb1, - int nb2, - int nb3, - int offset, + int ggml_fp32_to_fp16( + double arg0, ) { - return _ggml_set_inplace( - ctx, - a, - b, - nb1, - nb2, - nb3, - offset, + return _ggml_fp32_to_fp16( + arg0, ); } - late final _ggml_set_inplacePtr = _lookup< + late final _ggml_fp32_to_fp16Ptr = + _lookup>( + 'ggml_fp32_to_fp16'); + late final _ggml_fp32_to_fp16 = + _ggml_fp32_to_fp16Ptr.asFunction(); + + void ggml_fp32_to_fp16_row( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, + ) { + return _ggml_fp32_to_fp16_row( + arg0, + arg1, + arg2, + ); + } + + late final _ggml_fp32_to_fp16_rowPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_set_inplace'); - late final _ggml_set_inplace = _ggml_set_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('ggml_fp32_to_fp16_row'); + late final _ggml_fp32_to_fp16_row = _ggml_fp32_to_fp16_rowPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer ggml_set_1d( + void ggml_free( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int offset, ) { - return _ggml_set_1d( + return _ggml_free( ctx, - a, - b, - offset, ); } - late final _ggml_set_1dPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size)>>('ggml_set_1d'); - late final _ggml_set_1d = _ggml_set_1dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_freePtr = + _lookup)>>( + 'ggml_free'); + late final _ggml_free = + _ggml_freePtr.asFunction)>(); - ffi.Pointer ggml_set_1d_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int offset, + ggml_type ggml_ftype_to_ggml_type( + ggml_ftype ftype, ) { - return _ggml_set_1d_inplace( - ctx, - a, - b, - offset, - ); + return ggml_type.fromValue(_ggml_ftype_to_ggml_type( + ftype.value, + )); } - late final _ggml_set_1d_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size)>>('ggml_set_1d_inplace'); - late final _ggml_set_1d_inplace = _ggml_set_1d_inplacePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_ftype_to_ggml_typePtr = + _lookup>( + 'ggml_ftype_to_ggml_type'); + late final _ggml_ftype_to_ggml_type = + _ggml_ftype_to_ggml_typePtr.asFunction(); - ffi.Pointer ggml_set_2d( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int nb1, - int offset, + bool ggml_gallocr_alloc_graph( + ggml_gallocr_t galloc, + ffi.Pointer graph, ) { - return _ggml_set_2d( - ctx, - a, - b, - nb1, - offset, + return _ggml_gallocr_alloc_graph( + galloc, + graph, ); } - late final _ggml_set_2dPtr = _lookup< + late final _ggml_gallocr_alloc_graphPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size)>>('ggml_set_2d'); - late final _ggml_set_2d = _ggml_set_2dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Bool Function(ggml_gallocr_t, + ffi.Pointer)>>('ggml_gallocr_alloc_graph'); + late final _ggml_gallocr_alloc_graph = _ggml_gallocr_alloc_graphPtr + .asFunction)>(); - ffi.Pointer ggml_set_2d_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int nb1, - int offset, + void ggml_gallocr_free( + ggml_gallocr_t galloc, ) { - return _ggml_set_2d_inplace( - ctx, - a, - b, - nb1, - offset, + return _ggml_gallocr_free( + galloc, ); } - late final _ggml_set_2d_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size)>>('ggml_set_2d_inplace'); - late final _ggml_set_2d_inplace = _ggml_set_2d_inplacePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int)>(); + late final _ggml_gallocr_freePtr = + _lookup>( + 'ggml_gallocr_free'); + late final _ggml_gallocr_free = + _ggml_gallocr_freePtr.asFunction(); - ffi.Pointer ggml_cpy( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + int ggml_gallocr_get_buffer_size( + ggml_gallocr_t galloc, + int buffer_id, ) { - return _ggml_cpy( - ctx, - a, - b, + return _ggml_gallocr_get_buffer_size( + galloc, + buffer_id, ); } - late final _ggml_cpyPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>>('ggml_cpy'); - late final _ggml_cpy = _ggml_cpyPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_gallocr_get_buffer_sizePtr = + _lookup>( + 'ggml_gallocr_get_buffer_size'); + late final _ggml_gallocr_get_buffer_size = _ggml_gallocr_get_buffer_sizePtr + .asFunction(); - ffi.Pointer ggml_cast( - ffi.Pointer ctx, - ffi.Pointer a, - ggml_type type, + ggml_gallocr_t ggml_gallocr_new( + ggml_backend_buffer_type_t buft, ) { - return _ggml_cast( - ctx, - a, - type.value, + return _ggml_gallocr_new( + buft, ); } - late final _ggml_castPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.UnsignedInt)>>('ggml_cast'); - late final _ggml_cast = _ggml_castPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_gallocr_newPtr = _lookup< + ffi + .NativeFunction>( + 'ggml_gallocr_new'); + late final _ggml_gallocr_new = _ggml_gallocr_newPtr + .asFunction(); - ffi.Pointer ggml_cont( - ffi.Pointer ctx, - ffi.Pointer a, + ggml_gallocr_t ggml_gallocr_new_n( + ffi.Pointer bufts, + int n_bufs, ) { - return _ggml_cont( - ctx, - a, + return _ggml_gallocr_new_n( + bufts, + n_bufs, ); } - late final _ggml_contPtr = _lookup< + late final _ggml_gallocr_new_nPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_cont'); - late final _ggml_cont = _ggml_contPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ggml_gallocr_t Function(ffi.Pointer, + ffi.Int)>>('ggml_gallocr_new_n'); + late final _ggml_gallocr_new_n = _ggml_gallocr_new_nPtr.asFunction< + ggml_gallocr_t Function(ffi.Pointer, int)>(); - ffi.Pointer ggml_cont_1d( - ffi.Pointer ctx, - ffi.Pointer a, - int ne0, + bool ggml_gallocr_reserve( + ggml_gallocr_t galloc, + ffi.Pointer graph, ) { - return _ggml_cont_1d( - ctx, - a, - ne0, + return _ggml_gallocr_reserve( + galloc, + graph, ); } - late final _ggml_cont_1dPtr = _lookup< + late final _ggml_gallocr_reservePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int64)>>('ggml_cont_1d'); - late final _ggml_cont_1d = _ggml_cont_1dPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Bool Function(ggml_gallocr_t, + ffi.Pointer)>>('ggml_gallocr_reserve'); + late final _ggml_gallocr_reserve = _ggml_gallocr_reservePtr + .asFunction)>(); - ffi.Pointer ggml_cont_2d( - ffi.Pointer ctx, - ffi.Pointer a, - int ne0, - int ne1, + bool ggml_gallocr_reserve_n( + ggml_gallocr_t galloc, + ffi.Pointer graph, + ffi.Pointer node_buffer_ids, + ffi.Pointer leaf_buffer_ids, ) { - return _ggml_cont_2d( - ctx, - a, - ne0, - ne1, + return _ggml_gallocr_reserve_n( + galloc, + graph, + node_buffer_ids, + leaf_buffer_ids, ); } - late final _ggml_cont_2dPtr = _lookup< + late final _ggml_gallocr_reserve_nPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int64, ffi.Int64)>>('ggml_cont_2d'); - late final _ggml_cont_2d = _ggml_cont_2dPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Bool Function( + ggml_gallocr_t, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_gallocr_reserve_n'); + late final _ggml_gallocr_reserve_n = _ggml_gallocr_reserve_nPtr.asFunction< + bool Function(ggml_gallocr_t, ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_cont_3d( - ffi.Pointer ctx, - ffi.Pointer a, - int ne0, - int ne1, - int ne2, + void ggml_gallocr_reserve_n_size( + ggml_gallocr_t galloc, + ffi.Pointer graph, + ffi.Pointer node_buffer_ids, + ffi.Pointer leaf_buffer_ids, + ffi.Pointer sizes, ) { - return _ggml_cont_3d( - ctx, - a, - ne0, - ne1, - ne2, + return _ggml_gallocr_reserve_n_size( + galloc, + graph, + node_buffer_ids, + leaf_buffer_ids, + sizes, ); } - late final _ggml_cont_3dPtr = _lookup< + late final _ggml_gallocr_reserve_n_sizePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_cont_3d'); - late final _ggml_cont_3d = _ggml_cont_3dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int)>(); + ffi.Void Function( + ggml_gallocr_t, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_gallocr_reserve_n_size'); + late final _ggml_gallocr_reserve_n_size = + _ggml_gallocr_reserve_n_sizePtr.asFunction< + void Function( + ggml_gallocr_t, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - ffi.Pointer ggml_cont_4d( + ffi.Pointer ggml_gated_delta_net( ffi.Pointer ctx, - ffi.Pointer a, - int ne0, - int ne1, - int ne2, - int ne3, + ffi.Pointer q, + ffi.Pointer k, + ffi.Pointer v, + ffi.Pointer g, + ffi.Pointer beta, + ffi.Pointer state, + int K, ) { - return _ggml_cont_4d( + return _ggml_gated_delta_net( ctx, - a, - ne0, - ne1, - ne2, - ne3, + q, + k, + v, + g, + beta, + state, + K, ); } - late final _ggml_cont_4dPtr = _lookup< + late final _ggml_gated_delta_netPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_cont_4d'); - late final _ggml_cont_4d = _ggml_cont_4dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int64)>>('ggml_gated_delta_net'); + late final _ggml_gated_delta_net = _ggml_gated_delta_netPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int)>(); - ffi.Pointer ggml_reshape( + ffi.Pointer ggml_gated_linear_attn( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + ffi.Pointer k, + ffi.Pointer v, + ffi.Pointer q, + ffi.Pointer g, + ffi.Pointer state, + double scale, ) { - return _ggml_reshape( + return _ggml_gated_linear_attn( ctx, - a, - b, + k, + v, + q, + g, + state, + scale, ); } - late final _ggml_reshapePtr = _lookup< + late final _ggml_gated_linear_attnPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('ggml_reshape'); - late final _ggml_reshape = _ggml_reshapePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Float)>>('ggml_gated_linear_attn'); + late final _ggml_gated_linear_attn = _ggml_gated_linear_attnPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + double)>(); - ffi.Pointer ggml_reshape_1d( + ffi.Pointer ggml_geglu( ffi.Pointer ctx, ffi.Pointer a, - int ne0, ) { - return _ggml_reshape_1d( + return _ggml_geglu( ctx, a, - ne0, ); } - late final _ggml_reshape_1dPtr = _lookup< + late final _ggml_gegluPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int64)>>('ggml_reshape_1d'); - late final _ggml_reshape_1d = _ggml_reshape_1dPtr.asFunction< + ffi.Pointer)>>('ggml_geglu'); + late final _ggml_geglu = _ggml_gegluPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_reshape_2d( + ffi.Pointer ggml_geglu_erf( ffi.Pointer ctx, ffi.Pointer a, - int ne0, - int ne1, ) { - return _ggml_reshape_2d( + return _ggml_geglu_erf( ctx, a, - ne0, - ne1, ); } - late final _ggml_reshape_2dPtr = _lookup< + late final _ggml_geglu_erfPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int64, - ffi.Int64)>>('ggml_reshape_2d'); - late final _ggml_reshape_2d = _ggml_reshape_2dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_geglu_erf'); + late final _ggml_geglu_erf = _ggml_geglu_erfPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_reshape_3d( + ffi.Pointer ggml_geglu_erf_split( ffi.Pointer ctx, ffi.Pointer a, - int ne0, - int ne1, - int ne2, + ffi.Pointer b, ) { - return _ggml_reshape_3d( + return _ggml_geglu_erf_split( ctx, a, - ne0, - ne1, - ne2, + b, ); } - late final _ggml_reshape_3dPtr = _lookup< + late final _ggml_geglu_erf_splitPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_reshape_3d'); - late final _ggml_reshape_3d = _ggml_reshape_3dPtr.asFunction< + ffi.Pointer)>>('ggml_geglu_erf_split'); + late final _ggml_geglu_erf_split = _ggml_geglu_erf_splitPtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_reshape_4d( + ffi.Pointer ggml_geglu_erf_swapped( ffi.Pointer ctx, ffi.Pointer a, - int ne0, - int ne1, - int ne2, - int ne3, ) { - return _ggml_reshape_4d( + return _ggml_geglu_erf_swapped( ctx, a, - ne0, - ne1, - ne2, - ne3, ); } - late final _ggml_reshape_4dPtr = _lookup< + late final _ggml_geglu_erf_swappedPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_reshape_4d'); - late final _ggml_reshape_4d = _ggml_reshape_4dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_geglu_erf_swapped'); + late final _ggml_geglu_erf_swapped = _ggml_geglu_erf_swappedPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_view_1d( + ffi.Pointer ggml_geglu_quick( ffi.Pointer ctx, ffi.Pointer a, - int ne0, - int offset, ) { - return _ggml_view_1d( + return _ggml_geglu_quick( ctx, a, - ne0, - offset, ); } - late final _ggml_view_1dPtr = _lookup< + late final _ggml_geglu_quickPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int64, ffi.Size)>>('ggml_view_1d'); - late final _ggml_view_1d = _ggml_view_1dPtr.asFunction< + ffi.Pointer)>>('ggml_geglu_quick'); + late final _ggml_geglu_quick = _ggml_geglu_quickPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_view_2d( + ffi.Pointer ggml_geglu_quick_split( ffi.Pointer ctx, ffi.Pointer a, - int ne0, - int ne1, - int nb1, - int offset, + ffi.Pointer b, ) { - return _ggml_view_2d( + return _ggml_geglu_quick_split( ctx, a, - ne0, - ne1, - nb1, - offset, + b, ); } - late final _ggml_view_2dPtr = _lookup< + late final _ggml_geglu_quick_splitPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Size, - ffi.Size)>>('ggml_view_2d'); - late final _ggml_view_2d = _ggml_view_2dPtr.asFunction< + ffi.Pointer)>>('ggml_geglu_quick_split'); + late final _ggml_geglu_quick_split = _ggml_geglu_quick_splitPtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_view_3d( + ffi.Pointer ggml_geglu_quick_swapped( ffi.Pointer ctx, ffi.Pointer a, - int ne0, - int ne1, - int ne2, - int nb1, - int nb2, - int offset, ) { - return _ggml_view_3d( + return _ggml_geglu_quick_swapped( ctx, a, - ne0, - ne1, - ne2, - nb1, - nb2, - offset, ); } - late final _ggml_view_3dPtr = _lookup< + late final _ggml_geglu_quick_swappedPtr = _lookup< ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_geglu_quick_swapped'); + late final _ggml_geglu_quick_swapped = + _ggml_geglu_quick_swappedPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_view_3d'); - late final _ggml_view_3d = _ggml_view_3dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int, int, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_view_4d( + ffi.Pointer ggml_geglu_split( ffi.Pointer ctx, ffi.Pointer a, - int ne0, - int ne1, - int ne2, - int ne3, - int nb1, - int nb2, - int nb3, - int offset, + ffi.Pointer b, ) { - return _ggml_view_4d( + return _ggml_geglu_split( ctx, a, - ne0, - ne1, - ne2, - ne3, - nb1, - nb2, - nb3, - offset, + b, ); } - late final _ggml_view_4dPtr = _lookup< + late final _ggml_geglu_splitPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_view_4d'); - late final _ggml_view_4d = _ggml_view_4dPtr.asFunction< + ffi.Pointer)>>('ggml_geglu_split'); + late final _ggml_geglu_split = _ggml_geglu_splitPtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int, int, int, int, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_permute( + ffi.Pointer ggml_geglu_swapped( ffi.Pointer ctx, ffi.Pointer a, - int axis0, - int axis1, - int axis2, - int axis3, ) { - return _ggml_permute( + return _ggml_geglu_swapped( ctx, a, - axis0, - axis1, - axis2, - axis3, ); } - late final _ggml_permutePtr = _lookup< + late final _ggml_geglu_swappedPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_permute'); - late final _ggml_permute = _ggml_permutePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_geglu_swapped'); + late final _ggml_geglu_swapped = _ggml_geglu_swappedPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_transpose( + ffi.Pointer ggml_gelu( ffi.Pointer ctx, ffi.Pointer a, ) { - return _ggml_transpose( + return _ggml_gelu( ctx, a, ); } - late final _ggml_transposePtr = _lookup< + late final _ggml_geluPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_transpose'); - late final _ggml_transpose = _ggml_transposePtr.asFunction< + ffi.Pointer)>>('ggml_gelu'); + late final _ggml_gelu = _ggml_geluPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_get_rows( + ffi.Pointer ggml_gelu_erf( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, ) { - return _ggml_get_rows( + return _ggml_gelu_erf( ctx, a, - b, ); } - late final _ggml_get_rowsPtr = _lookup< + late final _ggml_gelu_erfPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_get_rows'); - late final _ggml_get_rows = _ggml_get_rowsPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_gelu_erf'); + late final _ggml_gelu_erf = _ggml_gelu_erfPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_get_rows_back( + ffi.Pointer ggml_gelu_erf_inplace( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, ) { - return _ggml_get_rows_back( + return _ggml_gelu_erf_inplace( ctx, a, - b, - c, ); } - late final _ggml_get_rows_backPtr = _lookup< + late final _ggml_gelu_erf_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_get_rows_back'); - late final _ggml_get_rows_back = _ggml_get_rows_backPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_gelu_erf_inplace'); + late final _ggml_gelu_erf_inplace = _ggml_gelu_erf_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_set_rows( + ffi.Pointer ggml_gelu_inplace( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, ) { - return _ggml_set_rows( + return _ggml_gelu_inplace( ctx, a, - b, - c, ); } - late final _ggml_set_rowsPtr = _lookup< + late final _ggml_gelu_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_set_rows'); - late final _ggml_set_rows = _ggml_set_rowsPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_gelu_inplace'); + late final _ggml_gelu_inplace = _ggml_gelu_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_diag( + ffi.Pointer ggml_gelu_quick( ffi.Pointer ctx, ffi.Pointer a, ) { - return _ggml_diag( + return _ggml_gelu_quick( ctx, a, ); } - late final _ggml_diagPtr = _lookup< + late final _ggml_gelu_quickPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_diag'); - late final _ggml_diag = _ggml_diagPtr.asFunction< + ffi.Pointer)>>('ggml_gelu_quick'); + late final _ggml_gelu_quick = _ggml_gelu_quickPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_diag_mask_inf( + ffi.Pointer ggml_gelu_quick_inplace( ffi.Pointer ctx, ffi.Pointer a, - int n_past, ) { - return _ggml_diag_mask_inf( + return _ggml_gelu_quick_inplace( ctx, a, - n_past, ); } - late final _ggml_diag_mask_infPtr = _lookup< + late final _ggml_gelu_quick_inplacePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int)>>('ggml_diag_mask_inf'); - late final _ggml_diag_mask_inf = _ggml_diag_mask_infPtr.asFunction< + ffi.Pointer)>>('ggml_gelu_quick_inplace'); + late final _ggml_gelu_quick_inplace = _ggml_gelu_quick_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_diag_mask_inf_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - int n_past, + ffi.Pointer ggml_get_data( + ffi.Pointer tensor, ) { - return _ggml_diag_mask_inf_inplace( - ctx, - a, - n_past, + return _ggml_get_data( + tensor, ); } - late final _ggml_diag_mask_inf_inplacePtr = _lookup< + late final _ggml_get_dataPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int)>>('ggml_diag_mask_inf_inplace'); - late final _ggml_diag_mask_inf_inplace = - _ggml_diag_mask_inf_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer Function( + ffi.Pointer)>>('ggml_get_data'); + late final _ggml_get_data = _ggml_get_dataPtr + .asFunction Function(ffi.Pointer)>(); - ffi.Pointer ggml_diag_mask_zero( - ffi.Pointer ctx, - ffi.Pointer a, - int n_past, + ffi.Pointer ggml_get_data_f32( + ffi.Pointer tensor, ) { - return _ggml_diag_mask_zero( - ctx, - a, - n_past, + return _ggml_get_data_f32( + tensor, ); } - late final _ggml_diag_mask_zeroPtr = _lookup< + late final _ggml_get_data_f32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int)>>('ggml_diag_mask_zero'); - late final _ggml_diag_mask_zero = _ggml_diag_mask_zeroPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer Function( + ffi.Pointer)>>('ggml_get_data_f32'); + late final _ggml_get_data_f32 = _ggml_get_data_f32Ptr + .asFunction Function(ffi.Pointer)>(); - ffi.Pointer ggml_diag_mask_zero_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - int n_past, + double ggml_get_f32_1d( + ffi.Pointer tensor, + int i, ) { - return _ggml_diag_mask_zero_inplace( - ctx, - a, - n_past, + return _ggml_get_f32_1d( + tensor, + i, ); } - late final _ggml_diag_mask_zero_inplacePtr = _lookup< + late final _ggml_get_f32_1dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int)>>('ggml_diag_mask_zero_inplace'); - late final _ggml_diag_mask_zero_inplace = - _ggml_diag_mask_zero_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Float Function( + ffi.Pointer, ffi.Int)>>('ggml_get_f32_1d'); + late final _ggml_get_f32_1d = _ggml_get_f32_1dPtr + .asFunction, int)>(); - ffi.Pointer ggml_soft_max( - ffi.Pointer ctx, - ffi.Pointer a, + double ggml_get_f32_nd( + ffi.Pointer tensor, + int i0, + int i1, + int i2, + int i3, ) { - return _ggml_soft_max( - ctx, - a, + return _ggml_get_f32_nd( + tensor, + i0, + i1, + i2, + i3, ); } - late final _ggml_soft_maxPtr = _lookup< + late final _ggml_get_f32_ndPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_soft_max'); - late final _ggml_soft_max = _ggml_soft_maxPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Float Function(ffi.Pointer, ffi.Int, ffi.Int, + ffi.Int, ffi.Int)>>('ggml_get_f32_nd'); + late final _ggml_get_f32_nd = _ggml_get_f32_ndPtr.asFunction< + double Function(ffi.Pointer, int, int, int, int)>(); - ffi.Pointer ggml_soft_max_inplace( + ffi.Pointer ggml_get_first_tensor( ffi.Pointer ctx, - ffi.Pointer a, ) { - return _ggml_soft_max_inplace( + return _ggml_get_first_tensor( ctx, - a, ); } - late final _ggml_soft_max_inplacePtr = _lookup< + late final _ggml_get_first_tensorPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_soft_max_inplace'); - late final _ggml_soft_max_inplace = _ggml_soft_max_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer)>>('ggml_get_first_tensor'); + late final _ggml_get_first_tensor = _ggml_get_first_tensorPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - ffi.Pointer ggml_soft_max_ext( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer mask, - double scale, - double max_bias, + ggml_glu_op ggml_get_glu_op( + ffi.Pointer tensor, ) { - return _ggml_soft_max_ext( - ctx, - a, - mask, - scale, - max_bias, - ); + return ggml_glu_op.fromValue(_ggml_get_glu_op( + tensor, + )); } - late final _ggml_soft_max_extPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float)>>('ggml_soft_max_ext'); - late final _ggml_soft_max_ext = _ggml_soft_max_extPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - double, - double)>(); + late final _ggml_get_glu_opPtr = _lookup< + ffi + .NativeFunction)>>( + 'ggml_get_glu_op'); + late final _ggml_get_glu_op = + _ggml_get_glu_opPtr.asFunction)>(); - ffi.Pointer ggml_soft_max_ext_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer mask, - double scale, - double max_bias, + int ggml_get_i32_1d( + ffi.Pointer tensor, + int i, ) { - return _ggml_soft_max_ext_inplace( - ctx, - a, - mask, - scale, - max_bias, + return _ggml_get_i32_1d( + tensor, + i, ); } - late final _ggml_soft_max_ext_inplacePtr = _lookup< + late final _ggml_get_i32_1dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float)>>('ggml_soft_max_ext_inplace'); - late final _ggml_soft_max_ext_inplace = - _ggml_soft_max_ext_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - double, - double)>(); + ffi.Int32 Function( + ffi.Pointer, ffi.Int)>>('ggml_get_i32_1d'); + late final _ggml_get_i32_1d = _ggml_get_i32_1dPtr + .asFunction, int)>(); - void ggml_soft_max_add_sinks( - ffi.Pointer a, - ffi.Pointer sinks, + int ggml_get_i32_nd( + ffi.Pointer tensor, + int i0, + int i1, + int i2, + int i3, ) { - return _ggml_soft_max_add_sinks( - a, - sinks, + return _ggml_get_i32_nd( + tensor, + i0, + i1, + i2, + i3, ); } - late final _ggml_soft_max_add_sinksPtr = _lookup< + late final _ggml_get_i32_ndPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('ggml_soft_max_add_sinks'); - late final _ggml_soft_max_add_sinks = _ggml_soft_max_add_sinksPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Int32 Function(ffi.Pointer, ffi.Int, ffi.Int, + ffi.Int, ffi.Int)>>('ggml_get_i32_nd'); + late final _ggml_get_i32_nd = _ggml_get_i32_ndPtr + .asFunction, int, int, int, int)>(); - ffi.Pointer ggml_soft_max_ext_back( + int ggml_get_max_tensor_size( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - double scale, - double max_bias, ) { - return _ggml_soft_max_ext_back( + return _ggml_get_max_tensor_size( ctx, - a, - b, - scale, - max_bias, ); } - late final _ggml_soft_max_ext_backPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float)>>('ggml_soft_max_ext_back'); - late final _ggml_soft_max_ext_back = _ggml_soft_max_ext_backPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - double, - double)>(); + late final _ggml_get_max_tensor_sizePtr = + _lookup)>>( + 'ggml_get_max_tensor_size'); + late final _ggml_get_max_tensor_size = _ggml_get_max_tensor_sizePtr + .asFunction)>(); - ffi.Pointer ggml_soft_max_ext_back_inplace( + ffi.Pointer ggml_get_mem_buffer( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - double scale, - double max_bias, ) { - return _ggml_soft_max_ext_back_inplace( + return _ggml_get_mem_buffer( ctx, - a, - b, - scale, - max_bias, ); } - late final _ggml_soft_max_ext_back_inplacePtr = _lookup< + late final _ggml_get_mem_bufferPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float)>>('ggml_soft_max_ext_back_inplace'); - late final _ggml_soft_max_ext_back_inplace = - _ggml_soft_max_ext_back_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - double, - double)>(); + ffi.Pointer Function( + ffi.Pointer)>>('ggml_get_mem_buffer'); + late final _ggml_get_mem_buffer = _ggml_get_mem_bufferPtr + .asFunction Function(ffi.Pointer)>(); - ffi.Pointer ggml_rope( + int ggml_get_mem_size( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int n_dims, - int mode, ) { - return _ggml_rope( + return _ggml_get_mem_size( ctx, - a, - b, - n_dims, - mode, ); } - late final _ggml_ropePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int)>>('ggml_rope'); - late final _ggml_rope = _ggml_ropePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int)>(); + late final _ggml_get_mem_sizePtr = + _lookup)>>( + 'ggml_get_mem_size'); + late final _ggml_get_mem_size = _ggml_get_mem_sizePtr + .asFunction)>(); - ffi.Pointer ggml_rope_inplace( + ffi.Pointer ggml_get_name( + ffi.Pointer tensor, + ) { + return _ggml_get_name( + tensor, + ); + } + + late final _ggml_get_namePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer)>>('ggml_get_name'); + late final _ggml_get_name = _ggml_get_namePtr + .asFunction Function(ffi.Pointer)>(); + + ffi.Pointer ggml_get_next_tensor( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int n_dims, - int mode, + ffi.Pointer tensor, ) { - return _ggml_rope_inplace( + return _ggml_get_next_tensor( ctx, - a, - b, - n_dims, - mode, + tensor, ); } - late final _ggml_rope_inplacePtr = _lookup< + late final _ggml_get_next_tensorPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int)>>('ggml_rope_inplace'); - late final _ggml_rope_inplace = _ggml_rope_inplacePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_get_next_tensor'); + late final _ggml_get_next_tensor = _ggml_get_next_tensorPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_rope_ext( + bool ggml_get_no_alloc( + ffi.Pointer ctx, + ) { + return _ggml_get_no_alloc( + ctx, + ); + } + + late final _ggml_get_no_allocPtr = + _lookup)>>( + 'ggml_get_no_alloc'); + late final _ggml_get_no_alloc = _ggml_get_no_allocPtr + .asFunction)>(); + + ffi.Pointer ggml_get_rel_pos( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, - int n_dims, - int mode, - int n_ctx_orig, - double freq_base, - double freq_scale, - double ext_factor, - double attn_factor, - double beta_fast, - double beta_slow, + int qh, + int kh, ) { - return _ggml_rope_ext( + return _ggml_get_rel_pos( ctx, a, - b, - c, - n_dims, - mode, - n_ctx_orig, - freq_base, - freq_scale, - ext_factor, - attn_factor, - beta_fast, - beta_slow, + qh, + kh, ); } - late final _ggml_rope_extPtr = _lookup< + late final _ggml_get_rel_posPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_rope_ext'); - late final _ggml_rope_ext = _ggml_rope_extPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Int, ffi.Int)>>('ggml_get_rel_pos'); + late final _ggml_get_rel_pos = _ggml_get_rel_posPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - double, - double, - double, - double, - double, - double)>(); + ffi.Pointer, ffi.Pointer, int, int)>(); - ffi.Pointer ggml_rope_multi( + ffi.Pointer ggml_get_rows( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, - ffi.Pointer c, - int n_dims, - ffi.Pointer sections, - int mode, - int n_ctx_orig, - double freq_base, - double freq_scale, - double ext_factor, - double attn_factor, - double beta_fast, - double beta_slow, ) { - return _ggml_rope_multi( + return _ggml_get_rows( ctx, a, b, - c, - n_dims, - sections, - mode, - n_ctx_orig, - freq_base, - freq_scale, - ext_factor, - attn_factor, - beta_fast, - beta_slow, ); } - late final _ggml_rope_multiPtr = _lookup< + late final _ggml_get_rowsPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_rope_multi'); - late final _ggml_rope_multi = _ggml_rope_multiPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - ffi.Pointer, - int, - int, - double, - double, - double, - double, - double, - double)>(); + ffi.Pointer)>>('ggml_get_rows'); + late final _ggml_get_rows = _ggml_get_rowsPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_rope_ext_inplace( + ffi.Pointer ggml_get_rows_back( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, ffi.Pointer c, - int n_dims, - int mode, - int n_ctx_orig, - double freq_base, - double freq_scale, - double ext_factor, - double attn_factor, - double beta_fast, - double beta_slow, ) { - return _ggml_rope_ext_inplace( + return _ggml_get_rows_back( ctx, a, b, c, - n_dims, - mode, - n_ctx_orig, - freq_base, - freq_scale, - ext_factor, - attn_factor, - beta_fast, - beta_slow, ); } - late final _ggml_rope_ext_inplacePtr = _lookup< + late final _ggml_get_rows_backPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_rope_ext_inplace'); - late final _ggml_rope_ext_inplace = _ggml_rope_ext_inplacePtr.asFunction< + ffi.Pointer)>>('ggml_get_rows_back'); + late final _ggml_get_rows_back = _ggml_get_rows_backPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, - ffi.Pointer, - int, - int, - int, - double, - double, - double, - double, - double, - double)>(); + ffi.Pointer)>(); - ffi.Pointer ggml_rope_multi_inplace( + ffi.Pointer ggml_get_tensor( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, - int n_dims, - ffi.Pointer sections, - int mode, - int n_ctx_orig, - double freq_base, - double freq_scale, - double ext_factor, - double attn_factor, - double beta_fast, - double beta_slow, + ffi.Pointer name, ) { - return _ggml_rope_multi_inplace( + return _ggml_get_tensor( ctx, - a, - b, - c, - n_dims, - sections, - mode, - n_ctx_orig, - freq_base, - freq_scale, - ext_factor, - attn_factor, - beta_fast, - beta_slow, + name, ); } - late final _ggml_rope_multi_inplacePtr = _lookup< + late final _ggml_get_tensorPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_rope_multi_inplace'); - late final _ggml_rope_multi_inplace = _ggml_rope_multi_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_get_tensor'); + late final _ggml_get_tensor = _ggml_get_tensorPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - ffi.Pointer, - int, - int, - double, - double, - double, - double, - double, - double)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_rope_custom( + ffi.Pointer ggml_get_type_traits( + ggml_type type, + ) { + return _ggml_get_type_traits( + type.value, + ); + } + + late final _ggml_get_type_traitsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.UnsignedInt)>>('ggml_get_type_traits'); + late final _ggml_get_type_traits = _ggml_get_type_traitsPtr + .asFunction Function(int)>(); + + ffi.Pointer ggml_get_type_traits_cpu( + ggml_type type, + ) { + return _ggml_get_type_traits_cpu( + type.value, + ); + } + + late final _ggml_get_type_traits_cpuPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.UnsignedInt)>>('ggml_get_type_traits_cpu'); + late final _ggml_get_type_traits_cpu = _ggml_get_type_traits_cpuPtr + .asFunction Function(int)>(); + + ggml_unary_op ggml_get_unary_op( + ffi.Pointer tensor, + ) { + return ggml_unary_op.fromValue(_ggml_get_unary_op( + tensor, + )); + } + + late final _ggml_get_unary_opPtr = _lookup< + ffi + .NativeFunction)>>( + 'ggml_get_unary_op'); + late final _ggml_get_unary_op = _ggml_get_unary_opPtr + .asFunction)>(); + + ffi.Pointer ggml_glu( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, - int n_dims, - int mode, - int n_ctx_orig, - double freq_base, - double freq_scale, - double ext_factor, - double attn_factor, - double beta_fast, - double beta_slow, + ggml_glu_op op, + bool swapped, ) { - return _ggml_rope_custom( + return _ggml_glu( ctx, a, - b, - n_dims, - mode, - n_ctx_orig, - freq_base, - freq_scale, - ext_factor, - attn_factor, - beta_fast, - beta_slow, + op.value, + swapped, ); } - late final _ggml_rope_customPtr = _lookup< + late final _ggml_gluPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_rope_custom'); - late final _ggml_rope_custom = _ggml_rope_customPtr.asFunction< + ffi.UnsignedInt, + ffi.Bool)>>('ggml_glu'); + late final _ggml_glu = _ggml_gluPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - double, - double, - double, - double, - double, - double)>(); + ffi.Pointer, ffi.Pointer, int, bool)>(); - ffi.Pointer ggml_rope_custom_inplace( + ffi.Pointer ggml_glu_op_name( + ggml_glu_op op, + ) { + return _ggml_glu_op_name( + op.value, + ); + } + + late final _ggml_glu_op_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'ggml_glu_op_name'); + late final _ggml_glu_op_name = + _ggml_glu_op_namePtr.asFunction Function(int)>(); + + ffi.Pointer ggml_glu_split( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, - int n_dims, - int mode, - int n_ctx_orig, - double freq_base, - double freq_scale, - double ext_factor, - double attn_factor, - double beta_fast, - double beta_slow, + ggml_glu_op op, ) { - return _ggml_rope_custom_inplace( + return _ggml_glu_split( ctx, a, b, - n_dims, - mode, - n_ctx_orig, - freq_base, - freq_scale, - ext_factor, - attn_factor, - beta_fast, - beta_slow, + op.value, ); } - late final _ggml_rope_custom_inplacePtr = _lookup< + late final _ggml_glu_splitPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_rope_custom_inplace'); - late final _ggml_rope_custom_inplace = - _ggml_rope_custom_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - double, - double, - double, - double, - double, - double)>(); + ffi.UnsignedInt)>>('ggml_glu_split'); + late final _ggml_glu_split = _ggml_glu_splitPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int)>(); - void ggml_rope_yarn_corr_dims( - int n_dims, - int n_ctx_orig, - double freq_base, - double beta_fast, - double beta_slow, - ffi.Pointer dims, + void ggml_graph_add_node( + ffi.Pointer cgraph, + ffi.Pointer tensor, ) { - return _ggml_rope_yarn_corr_dims( - n_dims, - n_ctx_orig, - freq_base, - beta_fast, - beta_slow, - dims, + return _ggml_graph_add_node( + cgraph, + tensor, ); } - late final _ggml_rope_yarn_corr_dimsPtr = _lookup< + late final _ggml_graph_add_nodePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Int, ffi.Int, ffi.Float, ffi.Float, ffi.Float, - ffi.Pointer)>>('ggml_rope_yarn_corr_dims'); - late final _ggml_rope_yarn_corr_dims = - _ggml_rope_yarn_corr_dimsPtr.asFunction< - void Function( - int, int, double, double, double, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('ggml_graph_add_node'); + late final _ggml_graph_add_node = _ggml_graph_add_nodePtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_rope_ext_back( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, - int n_dims, - int mode, - int n_ctx_orig, - double freq_base, - double freq_scale, - double ext_factor, - double attn_factor, - double beta_fast, - double beta_slow, + void ggml_graph_clear( + ffi.Pointer cgraph, ) { - return _ggml_rope_ext_back( - ctx, - a, - b, - c, - n_dims, - mode, - n_ctx_orig, - freq_base, - freq_scale, - ext_factor, - attn_factor, - beta_fast, - beta_slow, + return _ggml_graph_clear( + cgraph, ); } - late final _ggml_rope_ext_backPtr = _lookup< + late final _ggml_graph_clearPtr = + _lookup)>>( + 'ggml_graph_clear'); + late final _ggml_graph_clear = _ggml_graph_clearPtr + .asFunction)>(); + + ggml_status ggml_graph_compute( + ffi.Pointer cgraph, + ffi.Pointer cplan, + ) { + return ggml_status.fromValue(_ggml_graph_compute( + cgraph, + cplan, + )); + } + + late final _ggml_graph_computePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_rope_ext_back'); - late final _ggml_rope_ext_back = _ggml_rope_ext_backPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - double, - double, - double, - double, - double, - double)>(); + ffi.Int Function(ffi.Pointer, + ffi.Pointer)>>('ggml_graph_compute'); + late final _ggml_graph_compute = _ggml_graph_computePtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_rope_multi_back( + ggml_status ggml_graph_compute_with_ctx( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, - int n_dims, - ffi.Pointer sections, - int mode, - int n_ctx_orig, - double freq_base, - double freq_scale, - double ext_factor, - double attn_factor, - double beta_fast, - double beta_slow, + ffi.Pointer cgraph, + int n_threads, ) { - return _ggml_rope_multi_back( + return ggml_status.fromValue(_ggml_graph_compute_with_ctx( ctx, - a, - b, - c, - n_dims, - sections, - mode, - n_ctx_orig, - freq_base, - freq_scale, - ext_factor, - attn_factor, - beta_fast, - beta_slow, + cgraph, + n_threads, + )); + } + + late final _ggml_graph_compute_with_ctxPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + ffi.Int)>>('ggml_graph_compute_with_ctx'); + late final _ggml_graph_compute_with_ctx = + _ggml_graph_compute_with_ctxPtr.asFunction< + int Function( + ffi.Pointer, ffi.Pointer, int)>(); + + void ggml_graph_cpy( + ffi.Pointer src, + ffi.Pointer dst, + ) { + return _ggml_graph_cpy( + src, + dst, ); } - late final _ggml_rope_multi_backPtr = _lookup< + late final _ggml_graph_cpyPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_rope_multi_back'); - late final _ggml_rope_multi_back = _ggml_rope_multi_backPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - ffi.Pointer, - int, - int, - double, - double, - double, - double, - double, - double)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('ggml_graph_cpy'); + late final _ggml_graph_cpy = _ggml_graph_cpyPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_clamp( - ffi.Pointer ctx, - ffi.Pointer a, - double min, - double max, + void ggml_graph_dump_dot( + ffi.Pointer gb, + ffi.Pointer cgraph, + ffi.Pointer filename, ) { - return _ggml_clamp( - ctx, - a, - min, - max, + return _ggml_graph_dump_dot( + gb, + cgraph, + filename, ); } - late final _ggml_clampPtr = _lookup< + late final _ggml_graph_dump_dotPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float, ffi.Float)>>('ggml_clamp'); - late final _ggml_clamp = _ggml_clampPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, double, double)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer)>>('ggml_graph_dump_dot'); + late final _ggml_graph_dump_dot = _ggml_graph_dump_dotPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer)>(); - ffi.Pointer ggml_im2col( + ffi.Pointer ggml_graph_dup( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s0, - int s1, - int p0, - int p1, - int d0, - int d1, - bool is_2D, - ggml_type dst_type, + ffi.Pointer cgraph, + bool force_grads, ) { - return _ggml_im2col( + return _ggml_graph_dup( ctx, - a, - b, - s0, - s1, - p0, - p1, - d0, - d1, - is_2D, - dst_type.value, + cgraph, + force_grads, ); } - late final _ggml_im2colPtr = _lookup< + late final _ggml_graph_dupPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Bool, - ffi.UnsignedInt)>>('ggml_im2col'); - late final _ggml_im2col = _ggml_im2colPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int, - int, - int, - bool, - int)>(); - - ffi.Pointer ggml_im2col_back( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer ne, - int s0, - int s1, - int p0, - int p1, - int d0, - int d1, - bool is_2D, + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Bool)>>('ggml_graph_dup'); + late final _ggml_graph_dup = _ggml_graph_dupPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, bool)>(); + + ffi.Pointer ggml_graph_get_grad( + ffi.Pointer cgraph, + ffi.Pointer node, ) { - return _ggml_im2col_back( - ctx, - a, - b, - ne, - s0, - s1, - p0, - p1, - d0, - d1, - is_2D, + return _ggml_graph_get_grad( + cgraph, + node, ); } - late final _ggml_im2col_backPtr = _lookup< + late final _ggml_graph_get_gradPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Bool)>>('ggml_im2col_back'); - late final _ggml_im2col_back = _ggml_im2col_backPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_graph_get_grad'); + late final _ggml_graph_get_grad = _ggml_graph_get_gradPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int, - int, - int, - bool)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_col2im_1d( - ffi.Pointer ctx, - ffi.Pointer a, - int s0, - int oc, - int p0, + ffi.Pointer ggml_graph_get_grad_acc( + ffi.Pointer cgraph, + ffi.Pointer node, ) { - return _ggml_col2im_1d( - ctx, - a, - s0, - oc, - p0, + return _ggml_graph_get_grad_acc( + cgraph, + node, ); } - late final _ggml_col2im_1dPtr = _lookup< + late final _ggml_graph_get_grad_accPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_col2im_1d'); - late final _ggml_col2im_1d = _ggml_col2im_1dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_graph_get_grad_acc'); + late final _ggml_graph_get_grad_acc = _ggml_graph_get_grad_accPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_conv_1d( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s0, - int p0, - int d0, + ffi.Pointer ggml_graph_get_tensor( + ffi.Pointer cgraph, + ffi.Pointer name, ) { - return _ggml_conv_1d( - ctx, - a, - b, - s0, - p0, - d0, + return _ggml_graph_get_tensor( + cgraph, + name, ); } - late final _ggml_conv_1dPtr = _lookup< + late final _ggml_graph_get_tensorPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_conv_1d'); - late final _ggml_conv_1d = _ggml_conv_1dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_graph_get_tensor'); + late final _ggml_graph_get_tensor = _ggml_graph_get_tensorPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_conv_1d_ph( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s, - int d, + int ggml_graph_n_nodes( + ffi.Pointer cgraph, ) { - return _ggml_conv_1d_ph( - ctx, - a, - b, - s, - d, + return _ggml_graph_n_nodes( + cgraph, ); } - late final _ggml_conv_1d_phPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int)>>('ggml_conv_1d_ph'); - late final _ggml_conv_1d_ph = _ggml_conv_1d_phPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int)>(); + late final _ggml_graph_n_nodesPtr = + _lookup)>>( + 'ggml_graph_n_nodes'); + late final _ggml_graph_n_nodes = _ggml_graph_n_nodesPtr + .asFunction)>(); - ffi.Pointer ggml_conv_1d_dw( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s0, - int p0, - int d0, + ffi.Pointer ggml_graph_node( + ffi.Pointer cgraph, + int i, ) { - return _ggml_conv_1d_dw( - ctx, - a, - b, - s0, - p0, - d0, + return _ggml_graph_node( + cgraph, + i, ); } - late final _ggml_conv_1d_dwPtr = _lookup< + late final _ggml_graph_nodePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_conv_1d_dw'); - late final _ggml_conv_1d_dw = _ggml_conv_1d_dwPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int, int)>(); + ffi.Pointer, ffi.Int)>>('ggml_graph_node'); + late final _ggml_graph_node = _ggml_graph_nodePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - ffi.Pointer ggml_conv_1d_dw_ph( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s0, - int d0, + ffi.Pointer> ggml_graph_nodes( + ffi.Pointer cgraph, ) { - return _ggml_conv_1d_dw_ph( - ctx, - a, - b, - s0, - d0, + return _ggml_graph_nodes( + cgraph, ); } - late final _ggml_conv_1d_dw_phPtr = _lookup< + late final _ggml_graph_nodesPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int)>>('ggml_conv_1d_dw_ph'); - late final _ggml_conv_1d_dw_ph = _ggml_conv_1d_dw_phPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer> Function( + ffi.Pointer)>>('ggml_graph_nodes'); + late final _ggml_graph_nodes = _ggml_graph_nodesPtr.asFunction< + ffi.Pointer> Function( + ffi.Pointer)>(); - ffi.Pointer ggml_conv_transpose_1d( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s0, - int p0, - int d0, - ) { - return _ggml_conv_transpose_1d( - ctx, - a, - b, - s0, - p0, - d0, - ); + int ggml_graph_overhead() { + return _ggml_graph_overhead(); } - late final _ggml_conv_transpose_1dPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, + late final _ggml_graph_overheadPtr = + _lookup>('ggml_graph_overhead'); + late final _ggml_graph_overhead = + _ggml_graph_overheadPtr.asFunction(); + + int ggml_graph_overhead_custom( + int size, + bool grads, + ) { + return _ggml_graph_overhead_custom( + size, + grads, + ); + } + + late final _ggml_graph_overhead_customPtr = + _lookup>( + 'ggml_graph_overhead_custom'); + late final _ggml_graph_overhead_custom = + _ggml_graph_overhead_customPtr.asFunction(); + + ggml_cplan ggml_graph_plan( + ffi.Pointer cgraph, + int n_threads, + ffi.Pointer threadpool, + ) { + return _ggml_graph_plan( + cgraph, + n_threads, + threadpool, + ); + } + + late final _ggml_graph_planPtr = _lookup< + ffi.NativeFunction< + ggml_cplan Function(ffi.Pointer, ffi.Int, + ffi.Pointer)>>('ggml_graph_plan'); + late final _ggml_graph_plan = _ggml_graph_planPtr.asFunction< + ggml_cplan Function( + ffi.Pointer, int, ffi.Pointer)>(); + + void ggml_graph_print( + ffi.Pointer cgraph, + ) { + return _ggml_graph_print( + cgraph, + ); + } + + late final _ggml_graph_printPtr = + _lookup)>>( + 'ggml_graph_print'); + late final _ggml_graph_print = _ggml_graph_printPtr + .asFunction)>(); + + void ggml_graph_reset( + ffi.Pointer cgraph, + ) { + return _ggml_graph_reset( + cgraph, + ); + } + + late final _ggml_graph_resetPtr = + _lookup)>>( + 'ggml_graph_reset'); + late final _ggml_graph_reset = _ggml_graph_resetPtr + .asFunction)>(); + + int ggml_graph_size( + ffi.Pointer cgraph, + ) { + return _ggml_graph_size( + cgraph, + ); + } + + late final _ggml_graph_sizePtr = + _lookup)>>( + 'ggml_graph_size'); + late final _ggml_graph_size = + _ggml_graph_sizePtr.asFunction)>(); + + ffi.Pointer ggml_group_norm( + ffi.Pointer ctx, + ffi.Pointer a, + int n_groups, + double eps, + ) { + return _ggml_group_norm( + ctx, + a, + n_groups, + eps, + ); + } + + late final _ggml_group_normPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, ffi.Int, + ffi.Float)>>('ggml_group_norm'); + late final _ggml_group_norm = _ggml_group_normPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int, double)>(); + + ffi.Pointer ggml_group_norm_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + int n_groups, + double eps, + ) { + return _ggml_group_norm_inplace( + ctx, + a, + n_groups, + eps, + ); + } + + late final _ggml_group_norm_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, ffi.Int, - ffi.Int)>>('ggml_conv_transpose_1d'); - late final _ggml_conv_transpose_1d = _ggml_conv_transpose_1dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int, int, int)>(); + ffi.Float)>>('ggml_group_norm_inplace'); + late final _ggml_group_norm_inplace = _ggml_group_norm_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int, double)>(); - ffi.Pointer ggml_conv_2d( + bool ggml_guid_matches( + ggml_guid_t guid_a, + ggml_guid_t guid_b, + ) { + return _ggml_guid_matches( + guid_a, + guid_b, + ); + } + + late final _ggml_guid_matchesPtr = + _lookup>( + 'ggml_guid_matches'); + late final _ggml_guid_matches = _ggml_guid_matchesPtr + .asFunction(); + + ffi.Pointer ggml_hardsigmoid( + ffi.Pointer ctx, + ffi.Pointer a, + ) { + return _ggml_hardsigmoid( + ctx, + a, + ); + } + + late final _ggml_hardsigmoidPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_hardsigmoid'); + late final _ggml_hardsigmoid = _ggml_hardsigmoidPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); + + ffi.Pointer ggml_hardswish( + ffi.Pointer ctx, + ffi.Pointer a, + ) { + return _ggml_hardswish( + ctx, + a, + ); + } + + late final _ggml_hardswishPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_hardswish'); + late final _ggml_hardswish = _ggml_hardswishPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); + + ffi.Pointer ggml_im2col( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, @@ -7219,8 +7251,10 @@ class LlamaBindings { int p1, int d0, int d1, + bool is_2D, + ggml_type dst_type, ) { - return _ggml_conv_2d( + return _ggml_im2col( ctx, a, b, @@ -7230,10 +7264,12 @@ class LlamaBindings { p1, d0, d1, + is_2D, + dst_type.value, ); } - late final _ggml_conv_2dPtr = _lookup< + late final _ggml_im2colPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, @@ -7244,8 +7280,10 @@ class LlamaBindings { ffi.Int, ffi.Int, ffi.Int, - ffi.Int)>>('ggml_conv_2d'); - late final _ggml_conv_2d = _ggml_conv_2dPtr.asFunction< + ffi.Int, + ffi.Bool, + ffi.UnsignedInt)>>('ggml_im2col'); + late final _ggml_im2col = _ggml_im2colPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, @@ -7255,6 +7293,8 @@ class LlamaBindings { int, int, int, + int, + bool, int)>(); ffi.Pointer ggml_im2col_3d( @@ -7325,1756 +7365,1207 @@ class LlamaBindings { int, int)>(); - ffi.Pointer ggml_conv_3d( + ffi.Pointer ggml_im2col_back( ffi.Pointer ctx, ffi.Pointer a, ffi.Pointer b, - int IC, + ffi.Pointer ne, int s0, int s1, - int s2, int p0, int p1, - int p2, int d0, int d1, - int d2, + bool is_2D, ) { - return _ggml_conv_3d( + return _ggml_im2col_back( ctx, a, b, - IC, + ne, s0, s1, - s2, p0, p1, - p2, d0, d1, - d2, + is_2D, ); } - late final _ggml_conv_3dPtr = _lookup< + late final _ggml_im2col_backPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, - ffi.Int64, - ffi.Int, - ffi.Int, + ffi.Pointer, ffi.Int, ffi.Int, ffi.Int, ffi.Int, ffi.Int, ffi.Int, - ffi.Int)>>('ggml_conv_3d'); - late final _ggml_conv_3d = _ggml_conv_3dPtr.asFunction< + ffi.Bool)>>('ggml_im2col_back'); + late final _ggml_im2col_back = _ggml_im2col_backPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, + ffi.Pointer, int, int, int, int, int, int, - int, - int, - int, - int)>(); + bool)>(); - ffi.Pointer ggml_conv_2d_sk_p0( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, + ffi.Pointer ggml_init( + ggml_init_params params, ) { - return _ggml_conv_2d_sk_p0( - ctx, - a, - b, + return _ggml_init( + params, ); } - late final _ggml_conv_2d_sk_p0Ptr = _lookup< + late final _ggml_initPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_conv_2d_sk_p0'); - late final _ggml_conv_2d_sk_p0 = _ggml_conv_2d_sk_p0Ptr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ggml_init_params)>>('ggml_init'); + late final _ggml_init = _ggml_initPtr + .asFunction Function(ggml_init_params)>(); - ffi.Pointer ggml_conv_2d_s1_ph( + ffi.Pointer ggml_interpolate( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, + int ne0, + int ne1, + int ne2, + int ne3, + int mode, ) { - return _ggml_conv_2d_s1_ph( + return _ggml_interpolate( ctx, a, - b, + ne0, + ne1, + ne2, + ne3, + mode, ); } - late final _ggml_conv_2d_s1_phPtr = _lookup< + late final _ggml_interpolatePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('ggml_conv_2d_s1_ph'); - late final _ggml_conv_2d_s1_ph = _ggml_conv_2d_s1_phPtr.asFunction< + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Uint32)>>('ggml_interpolate'); + late final _ggml_interpolate = _ggml_interpolatePtr.asFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer, int, int, int, int, int)>(); - ffi.Pointer ggml_conv_2d_dw( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s0, - int s1, - int p0, - int p1, - int d0, - int d1, + bool ggml_is_3d( + ffi.Pointer tensor, ) { - return _ggml_conv_2d_dw( - ctx, - a, - b, - s0, - s1, - p0, - p1, - d0, - d1, + return _ggml_is_3d( + tensor, ); } - late final _ggml_conv_2d_dwPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_conv_2d_dw'); - late final _ggml_conv_2d_dw = _ggml_conv_2d_dwPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int, - int, - int)>(); + late final _ggml_is_3dPtr = + _lookup)>>( + 'ggml_is_3d'); + late final _ggml_is_3d = + _ggml_is_3dPtr.asFunction)>(); - ffi.Pointer ggml_conv_2d_dw_direct( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int stride0, - int stride1, - int pad0, - int pad1, - int dilation0, - int dilation1, + bool ggml_is_contiguous( + ffi.Pointer tensor, ) { - return _ggml_conv_2d_dw_direct( - ctx, - a, - b, - stride0, - stride1, - pad0, - pad1, - dilation0, - dilation1, + return _ggml_is_contiguous( + tensor, ); } - late final _ggml_conv_2d_dw_directPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_conv_2d_dw_direct'); - late final _ggml_conv_2d_dw_direct = _ggml_conv_2d_dw_directPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int, - int, - int)>(); + late final _ggml_is_contiguousPtr = + _lookup)>>( + 'ggml_is_contiguous'); + late final _ggml_is_contiguous = _ggml_is_contiguousPtr + .asFunction)>(); - ffi.Pointer ggml_conv_transpose_2d_p0( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int stride, + bool ggml_is_contiguous_0( + ffi.Pointer tensor, ) { - return _ggml_conv_transpose_2d_p0( - ctx, - a, - b, - stride, + return _ggml_is_contiguous_0( + tensor, ); } - late final _ggml_conv_transpose_2d_p0Ptr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int)>>('ggml_conv_transpose_2d_p0'); - late final _ggml_conv_transpose_2d_p0 = - _ggml_conv_transpose_2d_p0Ptr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_is_contiguous_0Ptr = + _lookup)>>( + 'ggml_is_contiguous_0'); + late final _ggml_is_contiguous_0 = _ggml_is_contiguous_0Ptr + .asFunction)>(); - ffi.Pointer ggml_conv_2d_direct( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s0, - int s1, - int p0, - int p1, - int d0, - int d1, + bool ggml_is_contiguous_1( + ffi.Pointer tensor, ) { - return _ggml_conv_2d_direct( - ctx, - a, - b, - s0, - s1, - p0, - p1, - d0, - d1, + return _ggml_is_contiguous_1( + tensor, ); } - late final _ggml_conv_2d_directPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_conv_2d_direct'); - late final _ggml_conv_2d_direct = _ggml_conv_2d_directPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int, - int, - int)>(); + late final _ggml_is_contiguous_1Ptr = + _lookup)>>( + 'ggml_is_contiguous_1'); + late final _ggml_is_contiguous_1 = _ggml_is_contiguous_1Ptr + .asFunction)>(); - ffi.Pointer ggml_conv_3d_direct( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - int s0, - int s1, - int s2, - int p0, - int p1, - int p2, - int d0, - int d1, - int d2, - int n_channels, - int n_batch, - int n_channels_out, + bool ggml_is_contiguous_2( + ffi.Pointer tensor, ) { - return _ggml_conv_3d_direct( - ctx, - a, - b, - s0, - s1, - s2, - p0, - p1, - p2, - d0, - d1, - d2, - n_channels, - n_batch, - n_channels_out, + return _ggml_is_contiguous_2( + tensor, ); } - late final _ggml_conv_3d_directPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_conv_3d_direct'); - late final _ggml_conv_3d_direct = _ggml_conv_3d_directPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int, - int, - int, - int, - int, - int, - int, - int, - int)>(); + late final _ggml_is_contiguous_2Ptr = + _lookup)>>( + 'ggml_is_contiguous_2'); + late final _ggml_is_contiguous_2 = _ggml_is_contiguous_2Ptr + .asFunction)>(); - ffi.Pointer ggml_pool_1d( - ffi.Pointer ctx, - ffi.Pointer a, - ggml_op_pool op, - int k0, - int s0, - int p0, + bool ggml_is_contiguous_channels( + ffi.Pointer tensor, ) { - return _ggml_pool_1d( - ctx, - a, - op.value, - k0, - s0, - p0, + return _ggml_is_contiguous_channels( + tensor, ); } - late final _ggml_pool_1dPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.UnsignedInt, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_pool_1d'); - late final _ggml_pool_1d = _ggml_pool_1dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + late final _ggml_is_contiguous_channelsPtr = + _lookup)>>( + 'ggml_is_contiguous_channels'); + late final _ggml_is_contiguous_channels = _ggml_is_contiguous_channelsPtr + .asFunction)>(); - ffi.Pointer ggml_pool_2d( - ffi.Pointer ctx, - ffi.Pointer a, - ggml_op_pool op, - int k0, - int k1, - int s0, - int s1, - double p0, - double p1, + bool ggml_is_contiguous_rows( + ffi.Pointer tensor, ) { - return _ggml_pool_2d( - ctx, - a, - op.value, - k0, - k1, - s0, - s1, - p0, - p1, + return _ggml_is_contiguous_rows( + tensor, ); } - late final _ggml_pool_2dPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.UnsignedInt, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float)>>('ggml_pool_2d'); - late final _ggml_pool_2d = _ggml_pool_2dPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int, int, double, double)>(); + late final _ggml_is_contiguous_rowsPtr = + _lookup)>>( + 'ggml_is_contiguous_rows'); + late final _ggml_is_contiguous_rows = _ggml_is_contiguous_rowsPtr + .asFunction)>(); - ffi.Pointer ggml_pool_2d_back( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer af, - ggml_op_pool op, - int k0, - int k1, - int s0, - int s1, - double p0, - double p1, + bool ggml_is_contiguous_to_1( + ffi.Pointer tensor, ) { - return _ggml_pool_2d_back( - ctx, - a, - af, - op.value, - k0, - k1, - s0, - s1, - p0, - p1, + return _ggml_is_contiguous_to_1( + tensor, ); } - late final _ggml_pool_2d_backPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.UnsignedInt, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Float, - ffi.Float)>>('ggml_pool_2d_back'); - late final _ggml_pool_2d_back = _ggml_pool_2d_backPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - int, - int, - double, - double)>(); + late final _ggml_is_contiguous_to_1Ptr = + _lookup)>>( + 'ggml_is_contiguous_to_1'); + late final _ggml_is_contiguous_to_1 = _ggml_is_contiguous_to_1Ptr + .asFunction)>(); - ffi.Pointer ggml_upscale( - ffi.Pointer ctx, - ffi.Pointer a, - int scale_factor, - ggml_scale_mode mode, + bool ggml_is_contiguous_to_2( + ffi.Pointer tensor, ) { - return _ggml_upscale( - ctx, - a, - scale_factor, - mode.value, + return _ggml_is_contiguous_to_2( + tensor, ); } - late final _ggml_upscalePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.UnsignedInt)>>('ggml_upscale'); - late final _ggml_upscale = _ggml_upscalePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + late final _ggml_is_contiguous_to_2Ptr = + _lookup)>>( + 'ggml_is_contiguous_to_2'); + late final _ggml_is_contiguous_to_2 = _ggml_is_contiguous_to_2Ptr + .asFunction)>(); - ffi.Pointer ggml_upscale_ext( - ffi.Pointer ctx, - ffi.Pointer a, - int ne0, - int ne1, - int ne2, - int ne3, - ggml_scale_mode mode, + bool ggml_is_contiguous_to_3( + ffi.Pointer tensor, ) { - return _ggml_upscale_ext( - ctx, - a, - ne0, - ne1, - ne2, - ne3, - mode.value, + return _ggml_is_contiguous_to_3( + tensor, ); } - late final _ggml_upscale_extPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.UnsignedInt)>>('ggml_upscale_ext'); - late final _ggml_upscale_ext = _ggml_upscale_extPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int, int)>(); + late final _ggml_is_contiguous_to_3Ptr = + _lookup)>>( + 'ggml_is_contiguous_to_3'); + late final _ggml_is_contiguous_to_3 = _ggml_is_contiguous_to_3Ptr + .asFunction)>(); - ffi.Pointer ggml_interpolate( - ffi.Pointer ctx, - ffi.Pointer a, - int ne0, - int ne1, - int ne2, - int ne3, - int mode, + bool ggml_is_contiguously_allocated( + ffi.Pointer tensor, ) { - return _ggml_interpolate( - ctx, - a, - ne0, - ne1, - ne2, - ne3, - mode, + return _ggml_is_contiguously_allocated( + tensor, ); } - late final _ggml_interpolatePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Uint32)>>('ggml_interpolate'); - late final _ggml_interpolate = _ggml_interpolatePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int, int)>(); + late final _ggml_is_contiguously_allocatedPtr = + _lookup)>>( + 'ggml_is_contiguously_allocated'); + late final _ggml_is_contiguously_allocated = + _ggml_is_contiguously_allocatedPtr + .asFunction)>(); - ffi.Pointer ggml_pad( - ffi.Pointer ctx, - ffi.Pointer a, - int p0, - int p1, - int p2, - int p3, + bool ggml_is_empty( + ffi.Pointer tensor, ) { - return _ggml_pad( - ctx, - a, - p0, - p1, - p2, - p3, + return _ggml_is_empty( + tensor, ); } - late final _ggml_padPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_pad'); - late final _ggml_pad = _ggml_padPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + late final _ggml_is_emptyPtr = + _lookup)>>( + 'ggml_is_empty'); + late final _ggml_is_empty = + _ggml_is_emptyPtr.asFunction)>(); - ffi.Pointer ggml_pad_circular( + bool ggml_is_matrix( + ffi.Pointer tensor, + ) { + return _ggml_is_matrix( + tensor, + ); + } + + late final _ggml_is_matrixPtr = + _lookup)>>( + 'ggml_is_matrix'); + late final _ggml_is_matrix = + _ggml_is_matrixPtr.asFunction)>(); + + bool ggml_is_numa() { + return _ggml_is_numa(); + } + + late final _ggml_is_numaPtr = + _lookup>('ggml_is_numa'); + late final _ggml_is_numa = _ggml_is_numaPtr.asFunction(); + + bool ggml_is_permuted( + ffi.Pointer tensor, + ) { + return _ggml_is_permuted( + tensor, + ); + } + + late final _ggml_is_permutedPtr = + _lookup)>>( + 'ggml_is_permuted'); + late final _ggml_is_permuted = _ggml_is_permutedPtr + .asFunction)>(); + + bool ggml_is_quantized( + ggml_type type, + ) { + return _ggml_is_quantized( + type.value, + ); + } + + late final _ggml_is_quantizedPtr = + _lookup>( + 'ggml_is_quantized'); + late final _ggml_is_quantized = + _ggml_is_quantizedPtr.asFunction(); + + bool ggml_is_scalar( + ffi.Pointer tensor, + ) { + return _ggml_is_scalar( + tensor, + ); + } + + late final _ggml_is_scalarPtr = + _lookup)>>( + 'ggml_is_scalar'); + late final _ggml_is_scalar = + _ggml_is_scalarPtr.asFunction)>(); + + bool ggml_is_transposed( + ffi.Pointer tensor, + ) { + return _ggml_is_transposed( + tensor, + ); + } + + late final _ggml_is_transposedPtr = + _lookup)>>( + 'ggml_is_transposed'); + late final _ggml_is_transposed = _ggml_is_transposedPtr + .asFunction)>(); + + bool ggml_is_vector( + ffi.Pointer tensor, + ) { + return _ggml_is_vector( + tensor, + ); + } + + late final _ggml_is_vectorPtr = + _lookup)>>( + 'ggml_is_vector'); + late final _ggml_is_vector = + _ggml_is_vectorPtr.asFunction)>(); + + bool ggml_is_view( + ffi.Pointer tensor, + ) { + return _ggml_is_view( + tensor, + ); + } + + late final _ggml_is_viewPtr = + _lookup)>>( + 'ggml_is_view'); + late final _ggml_is_view = + _ggml_is_viewPtr.asFunction)>(); + + ffi.Pointer ggml_l2_norm( ffi.Pointer ctx, ffi.Pointer a, - int p0, - int p1, - int p2, - int p3, + double eps, ) { - return _ggml_pad_circular( + return _ggml_l2_norm( ctx, a, - p0, - p1, - p2, - p3, + eps, ); } - late final _ggml_pad_circularPtr = _lookup< + late final _ggml_l2_normPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_pad_circular'); - late final _ggml_pad_circular = _ggml_pad_circularPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_l2_norm'); + late final _ggml_l2_norm = _ggml_l2_normPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, double)>(); - ffi.Pointer ggml_pad_ext( + ffi.Pointer ggml_l2_norm_inplace( ffi.Pointer ctx, ffi.Pointer a, - int lp0, - int rp0, - int lp1, - int rp1, - int lp2, - int rp2, - int lp3, - int rp3, + double eps, ) { - return _ggml_pad_ext( + return _ggml_l2_norm_inplace( ctx, a, - lp0, - rp0, - lp1, - rp1, - lp2, - rp2, - lp3, - rp3, + eps, ); } - late final _ggml_pad_extPtr = _lookup< + late final _ggml_l2_norm_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_pad_ext'); - late final _ggml_pad_ext = _ggml_pad_extPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int, int, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_l2_norm_inplace'); + late final _ggml_l2_norm_inplace = _ggml_l2_norm_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, double)>(); - ffi.Pointer ggml_pad_ext_circular( + ffi.Pointer ggml_leaky_relu( ffi.Pointer ctx, ffi.Pointer a, - int lp0, - int rp0, - int lp1, - int rp1, - int lp2, - int rp2, - int lp3, - int rp3, + double negative_slope, + bool inplace, ) { - return _ggml_pad_ext_circular( + return _ggml_leaky_relu( ctx, a, - lp0, - rp0, - lp1, - rp1, - lp2, - rp2, - lp3, - rp3, + negative_slope, + inplace, ); } - late final _ggml_pad_ext_circularPtr = _lookup< + late final _ggml_leaky_reluPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_pad_ext_circular'); - late final _ggml_pad_ext_circular = _ggml_pad_ext_circularPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int, int, int, int, int)>(); + ffi.Float, + ffi.Bool)>>('ggml_leaky_relu'); + late final _ggml_leaky_relu = _ggml_leaky_reluPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, double, bool)>(); - ffi.Pointer ggml_pad_reflect_1d( + ffi.Pointer ggml_lightning_indexer( ffi.Pointer ctx, - ffi.Pointer a, - int p0, - int p1, + ffi.Pointer q, + ffi.Pointer k, + ffi.Pointer weights, + ffi.Pointer mask, ) { - return _ggml_pad_reflect_1d( + return _ggml_lightning_indexer( ctx, - a, - p0, - p1, + q, + k, + weights, + mask, ); } - late final _ggml_pad_reflect_1dPtr = _lookup< + late final _ggml_lightning_indexerPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Int, - ffi.Int)>>('ggml_pad_reflect_1d'); - late final _ggml_pad_reflect_1d = _ggml_pad_reflect_1dPtr.asFunction< + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_lightning_indexer'); + late final _ggml_lightning_indexer = _ggml_lightning_indexerPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - ffi.Pointer ggml_roll( + ffi.Pointer ggml_log( ffi.Pointer ctx, ffi.Pointer a, - int shift0, - int shift1, - int shift2, - int shift3, ) { - return _ggml_roll( + return _ggml_log( ctx, a, - shift0, - shift1, - shift2, - shift3, ); } - late final _ggml_rollPtr = _lookup< + late final _ggml_logPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_roll'); - late final _ggml_roll = _ggml_rollPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_log'); + late final _ggml_log = _ggml_logPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_tri( - ffi.Pointer ctx, - ffi.Pointer a, - ggml_tri_type type, + void ggml_log_get( + ffi.Pointer log_callback, + ffi.Pointer> user_data, ) { - return _ggml_tri( - ctx, - a, - type.value, + return _ggml_log_get( + log_callback, + user_data, ); } - late final _ggml_triPtr = _lookup< + late final _ggml_log_getPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.UnsignedInt)>>('ggml_tri'); - late final _ggml_tri = _ggml_triPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer>)>>('ggml_log_get'); + late final _ggml_log_get = _ggml_log_getPtr.asFunction< + void Function(ffi.Pointer, + ffi.Pointer>)>(); - ffi.Pointer ggml_fill( + ffi.Pointer ggml_log_inplace( ffi.Pointer ctx, ffi.Pointer a, - double c, ) { - return _ggml_fill( + return _ggml_log_inplace( ctx, a, - c, ); } - late final _ggml_fillPtr = _lookup< + late final _ggml_log_inplacePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_fill'); - late final _ggml_fill = _ggml_fillPtr.asFunction< + ffi.Pointer)>>('ggml_log_inplace'); + late final _ggml_log_inplace = _ggml_log_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_fill_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - double c, + void ggml_log_set( + ggml_log_callback log_callback, + ffi.Pointer user_data, ) { - return _ggml_fill_inplace( - ctx, - a, - c, + return _ggml_log_set( + log_callback, + user_data, ); } - late final _ggml_fill_inplacePtr = _lookup< + late final _ggml_log_setPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Float)>>('ggml_fill_inplace'); - late final _ggml_fill_inplace = _ggml_fill_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Void Function( + ggml_log_callback, ffi.Pointer)>>('ggml_log_set'); + late final _ggml_log_set = _ggml_log_setPtr + .asFunction)>(); - ffi.Pointer ggml_timestep_embedding( + ffi.Pointer ggml_map_custom1( ffi.Pointer ctx, - ffi.Pointer timesteps, - int dim, - int max_period, + ffi.Pointer a, + ggml_custom1_op_t fun, + int n_tasks, + ffi.Pointer userdata, ) { - return _ggml_timestep_embedding( + return _ggml_map_custom1( ctx, - timesteps, - dim, - max_period, + a, + fun, + n_tasks, + userdata, ); } - late final _ggml_timestep_embeddingPtr = _lookup< + late final _ggml_map_custom1Ptr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, + ggml_custom1_op_t, ffi.Int, - ffi.Int)>>('ggml_timestep_embedding'); - late final _ggml_timestep_embedding = _ggml_timestep_embeddingPtr.asFunction< + ffi.Pointer)>>('ggml_map_custom1'); + late final _ggml_map_custom1 = _ggml_map_custom1Ptr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer, + ffi.Pointer, + ggml_custom1_op_t, + int, + ffi.Pointer)>(); - ffi.Pointer ggml_argsort( + ffi.Pointer ggml_map_custom1_inplace( ffi.Pointer ctx, ffi.Pointer a, - ggml_sort_order order, + ggml_custom1_op_t fun, + int n_tasks, + ffi.Pointer userdata, ) { - return _ggml_argsort( + return _ggml_map_custom1_inplace( ctx, a, - order.value, + fun, + n_tasks, + userdata, ); } - late final _ggml_argsortPtr = _lookup< + late final _ggml_map_custom1_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.UnsignedInt)>>('ggml_argsort'); - late final _ggml_argsort = _ggml_argsortPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ggml_custom1_op_t, + ffi.Int, + ffi.Pointer)>>('ggml_map_custom1_inplace'); + late final _ggml_map_custom1_inplace = + _ggml_map_custom1_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ggml_custom1_op_t, + int, + ffi.Pointer)>(); - ffi.Pointer ggml_argsort_top_k( + ffi.Pointer ggml_map_custom2( ffi.Pointer ctx, ffi.Pointer a, - int k, + ffi.Pointer b, + ggml_custom2_op_t fun, + int n_tasks, + ffi.Pointer userdata, ) { - return _ggml_argsort_top_k( + return _ggml_map_custom2( ctx, a, - k, + b, + fun, + n_tasks, + userdata, ); } - late final _ggml_argsort_top_kPtr = _lookup< + late final _ggml_map_custom2Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int)>>('ggml_argsort_top_k'); - late final _ggml_argsort_top_k = _ggml_argsort_top_kPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ggml_custom2_op_t, + ffi.Int, + ffi.Pointer)>>('ggml_map_custom2'); + late final _ggml_map_custom2 = _ggml_map_custom2Ptr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ggml_custom2_op_t, + int, + ffi.Pointer)>(); - ffi.Pointer ggml_top_k( + ffi.Pointer ggml_map_custom2_inplace( ffi.Pointer ctx, ffi.Pointer a, - int k, + ffi.Pointer b, + ggml_custom2_op_t fun, + int n_tasks, + ffi.Pointer userdata, ) { - return _ggml_top_k( + return _ggml_map_custom2_inplace( ctx, a, - k, + b, + fun, + n_tasks, + userdata, ); } - late final _ggml_top_kPtr = _lookup< + late final _ggml_map_custom2_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int)>>('ggml_top_k'); - late final _ggml_top_k = _ggml_top_kPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); - - ffi.Pointer ggml_arange( - ffi.Pointer ctx, - double start, - double stop, - double step, - ) { - return _ggml_arange( - ctx, - start, - stop, - step, - ); - } - - late final _ggml_arangePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Float, ffi.Float, ffi.Float)>>('ggml_arange'); - late final _ggml_arange = _ggml_arangePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, double, double, double)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ggml_custom2_op_t, + ffi.Int, + ffi.Pointer)>>('ggml_map_custom2_inplace'); + late final _ggml_map_custom2_inplace = + _ggml_map_custom2_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ggml_custom2_op_t, + int, + ffi.Pointer)>(); - ffi.Pointer ggml_flash_attn_ext( + ffi.Pointer ggml_map_custom3( ffi.Pointer ctx, - ffi.Pointer q, - ffi.Pointer k, - ffi.Pointer v, - ffi.Pointer mask, - double scale, - double max_bias, - double logit_softcap, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + ggml_custom3_op_t fun, + int n_tasks, + ffi.Pointer userdata, ) { - return _ggml_flash_attn_ext( + return _ggml_map_custom3( ctx, - q, - k, - v, - mask, - scale, - max_bias, - logit_softcap, + a, + b, + c, + fun, + n_tasks, + userdata, ); } - late final _ggml_flash_attn_extPtr = _lookup< + late final _ggml_map_custom3Ptr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Float, - ffi.Float, - ffi.Float)>>('ggml_flash_attn_ext'); - late final _ggml_flash_attn_ext = _ggml_flash_attn_extPtr.asFunction< + ggml_custom3_op_t, + ffi.Int, + ffi.Pointer)>>('ggml_map_custom3'); + late final _ggml_map_custom3 = _ggml_map_custom3Ptr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, ffi.Pointer, - ffi.Pointer, - double, - double, - double)>(); + ggml_custom3_op_t, + int, + ffi.Pointer)>(); - void ggml_flash_attn_ext_set_prec( + ffi.Pointer ggml_map_custom3_inplace( + ffi.Pointer ctx, ffi.Pointer a, - ggml_prec prec, + ffi.Pointer b, + ffi.Pointer c, + ggml_custom3_op_t fun, + int n_tasks, + ffi.Pointer userdata, ) { - return _ggml_flash_attn_ext_set_prec( + return _ggml_map_custom3_inplace( + ctx, a, - prec.value, + b, + c, + fun, + n_tasks, + userdata, ); } - late final _ggml_flash_attn_ext_set_precPtr = _lookup< + late final _ggml_map_custom3_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.UnsignedInt)>>('ggml_flash_attn_ext_set_prec'); - late final _ggml_flash_attn_ext_set_prec = _ggml_flash_attn_ext_set_precPtr - .asFunction, int)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ggml_custom3_op_t, + ffi.Int, + ffi.Pointer)>>('ggml_map_custom3_inplace'); + late final _ggml_map_custom3_inplace = + _ggml_map_custom3_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ggml_custom3_op_t, + int, + ffi.Pointer)>(); - ggml_prec ggml_flash_attn_ext_get_prec( + ffi.Pointer ggml_mean( + ffi.Pointer ctx, ffi.Pointer a, ) { - return ggml_prec.fromValue(_ggml_flash_attn_ext_get_prec( + return _ggml_mean( + ctx, a, - )); + ); } - late final _ggml_flash_attn_ext_get_precPtr = _lookup< - ffi - .NativeFunction)>>( - 'ggml_flash_attn_ext_get_prec'); - late final _ggml_flash_attn_ext_get_prec = _ggml_flash_attn_ext_get_precPtr - .asFunction)>(); + late final _ggml_meanPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_mean'); + late final _ggml_mean = _ggml_meanPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_flash_attn_ext_add_sinks( + ffi.Pointer ggml_mul( + ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer sinks, + ffi.Pointer b, ) { - return _ggml_flash_attn_ext_add_sinks( + return _ggml_mul( + ctx, a, - sinks, + b, ); } - late final _ggml_flash_attn_ext_add_sinksPtr = _lookup< + late final _ggml_mulPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('ggml_flash_attn_ext_add_sinks'); - late final _ggml_flash_attn_ext_add_sinks = - _ggml_flash_attn_ext_add_sinksPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>>('ggml_mul'); + late final _ggml_mul = _ggml_mulPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_flash_attn_back( + ffi.Pointer ggml_mul_inplace( ffi.Pointer ctx, - ffi.Pointer q, - ffi.Pointer k, - ffi.Pointer v, - ffi.Pointer d, - bool masked, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_flash_attn_back( + return _ggml_mul_inplace( ctx, - q, - k, - v, - d, - masked, + a, + b, ); } - late final _ggml_flash_attn_backPtr = _lookup< + late final _ggml_mul_inplacePtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Bool)>>('ggml_flash_attn_back'); - late final _ggml_flash_attn_back = _ggml_flash_attn_backPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - bool)>(); + ffi.Pointer)>>('ggml_mul_inplace'); + late final _ggml_mul_inplace = _ggml_mul_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_ssm_conv( + ffi.Pointer ggml_mul_mat( ffi.Pointer ctx, - ffi.Pointer sx, - ffi.Pointer c, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_ssm_conv( + return _ggml_mul_mat( ctx, - sx, - c, + a, + b, ); } - late final _ggml_ssm_convPtr = _lookup< + late final _ggml_mul_matPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('ggml_ssm_conv'); - late final _ggml_ssm_conv = _ggml_ssm_convPtr.asFunction< + ffi.Pointer)>>('ggml_mul_mat'); + late final _ggml_mul_mat = _ggml_mul_matPtr.asFunction< ffi.Pointer Function(ffi.Pointer, ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_ssm_scan( + ffi.Pointer ggml_mul_mat_id( ffi.Pointer ctx, - ffi.Pointer s, - ffi.Pointer x, - ffi.Pointer dt, - ffi.Pointer A, - ffi.Pointer B, - ffi.Pointer C, + ffi.Pointer as, + ffi.Pointer b, ffi.Pointer ids, ) { - return _ggml_ssm_scan( + return _ggml_mul_mat_id( ctx, - s, - x, - dt, - A, - B, - C, + as, + b, ids, ); } - late final _ggml_ssm_scanPtr = _lookup< + late final _ggml_mul_mat_idPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_ssm_scan'); - late final _ggml_ssm_scan = _ggml_ssm_scanPtr.asFunction< + ffi.Pointer)>>('ggml_mul_mat_id'); + late final _ggml_mul_mat_id = _ggml_mul_mat_idPtr.asFunction< ffi.Pointer Function( ffi.Pointer, ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_win_part( - ffi.Pointer ctx, + void ggml_mul_mat_set_hint( ffi.Pointer a, - int w, + ggml_op_hint hint, ) { - return _ggml_win_part( - ctx, + return _ggml_mul_mat_set_hint( a, - w, + hint.value, ); } - late final _ggml_win_partPtr = _lookup< + late final _ggml_mul_mat_set_hintPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int)>>('ggml_win_part'); - late final _ggml_win_part = _ggml_win_partPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + ffi.Void Function(ffi.Pointer, + ffi.UnsignedInt)>>('ggml_mul_mat_set_hint'); + late final _ggml_mul_mat_set_hint = _ggml_mul_mat_set_hintPtr + .asFunction, int)>(); - ffi.Pointer ggml_win_unpart( - ffi.Pointer ctx, + void ggml_mul_mat_set_prec( ffi.Pointer a, - int w0, - int h0, - int w, + ggml_prec prec, ) { - return _ggml_win_unpart( - ctx, + return _ggml_mul_mat_set_prec( a, - w0, - h0, - w, + prec.value, ); } - late final _ggml_win_unpartPtr = _lookup< + late final _ggml_mul_mat_set_precPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Int, - ffi.Int)>>('ggml_win_unpart'); - late final _ggml_win_unpart = _ggml_win_unpartPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, int, int, int)>(); + ffi.Void Function(ffi.Pointer, + ffi.UnsignedInt)>>('ggml_mul_mat_set_prec'); + late final _ggml_mul_mat_set_prec = _ggml_mul_mat_set_precPtr + .asFunction, int)>(); - ffi.Pointer ggml_unary( - ffi.Pointer ctx, - ffi.Pointer a, - ggml_unary_op op, + int ggml_n_dims( + ffi.Pointer tensor, ) { - return _ggml_unary( - ctx, - a, - op.value, + return _ggml_n_dims( + tensor, ); } - late final _ggml_unaryPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.UnsignedInt)>>('ggml_unary'); - late final _ggml_unary = _ggml_unaryPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_n_dimsPtr = + _lookup)>>( + 'ggml_n_dims'); + late final _ggml_n_dims = + _ggml_n_dimsPtr.asFunction)>(); - ffi.Pointer ggml_unary_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ggml_unary_op op, + int ggml_nbytes( + ffi.Pointer tensor, ) { - return _ggml_unary_inplace( - ctx, - a, - op.value, + return _ggml_nbytes( + tensor, ); } - late final _ggml_unary_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.UnsignedInt)>>('ggml_unary_inplace'); - late final _ggml_unary_inplace = _ggml_unary_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_nbytesPtr = + _lookup)>>( + 'ggml_nbytes'); + late final _ggml_nbytes = + _ggml_nbytesPtr.asFunction)>(); - ffi.Pointer ggml_get_rel_pos( + int ggml_nbytes_pad( + ffi.Pointer tensor, + ) { + return _ggml_nbytes_pad( + tensor, + ); + } + + late final _ggml_nbytes_padPtr = + _lookup)>>( + 'ggml_nbytes_pad'); + late final _ggml_nbytes_pad = + _ggml_nbytes_padPtr.asFunction)>(); + + ffi.Pointer ggml_neg( ffi.Pointer ctx, ffi.Pointer a, - int qh, - int kh, ) { - return _ggml_get_rel_pos( + return _ggml_neg( ctx, a, - qh, - kh, ); } - late final _ggml_get_rel_posPtr = _lookup< + late final _ggml_negPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Int, ffi.Int)>>('ggml_get_rel_pos'); - late final _ggml_get_rel_pos = _ggml_get_rel_posPtr.asFunction< + ffi.Pointer)>>('ggml_neg'); + late final _ggml_neg = _ggml_negPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_add_rel_pos( + ffi.Pointer ggml_neg_inplace( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer pw, - ffi.Pointer ph, ) { - return _ggml_add_rel_pos( + return _ggml_neg_inplace( ctx, a, - pw, - ph, ); } - late final _ggml_add_rel_posPtr = _lookup< + late final _ggml_neg_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_add_rel_pos'); - late final _ggml_add_rel_pos = _ggml_add_rel_posPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_neg_inplace'); + late final _ggml_neg_inplace = _ggml_neg_inplacePtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_add_rel_pos_inplace( + int ggml_nelements( + ffi.Pointer tensor, + ) { + return _ggml_nelements( + tensor, + ); + } + + late final _ggml_nelementsPtr = + _lookup)>>( + 'ggml_nelements'); + late final _ggml_nelements = + _ggml_nelementsPtr.asFunction)>(); + + ffi.Pointer ggml_new_buffer( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer pw, - ffi.Pointer ph, + int nbytes, ) { - return _ggml_add_rel_pos_inplace( + return _ggml_new_buffer( ctx, - a, - pw, - ph, + nbytes, ); } - late final _ggml_add_rel_pos_inplacePtr = _lookup< + late final _ggml_new_bufferPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_add_rel_pos_inplace'); - late final _ggml_add_rel_pos_inplace = - _ggml_add_rel_pos_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, ffi.Size)>>('ggml_new_buffer'); + late final _ggml_new_buffer = _ggml_new_bufferPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - ffi.Pointer ggml_rwkv_wkv6( + ffi.Pointer ggml_new_f32( ffi.Pointer ctx, - ffi.Pointer k, - ffi.Pointer v, - ffi.Pointer r, - ffi.Pointer tf, - ffi.Pointer td, - ffi.Pointer state, + double value, ) { - return _ggml_rwkv_wkv6( + return _ggml_new_f32( ctx, - k, - v, - r, - tf, - td, - state, + value, ); } - late final _ggml_rwkv_wkv6Ptr = _lookup< + late final _ggml_new_f32Ptr = _lookup< ffi.NativeFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_rwkv_wkv6'); - late final _ggml_rwkv_wkv6 = _ggml_rwkv_wkv6Ptr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer, ffi.Float)>>('ggml_new_f32'); + late final _ggml_new_f32 = _ggml_new_f32Ptr.asFunction< + ffi.Pointer Function(ffi.Pointer, double)>(); - ffi.Pointer ggml_gated_linear_attn( + ffi.Pointer ggml_new_graph( ffi.Pointer ctx, - ffi.Pointer k, - ffi.Pointer v, - ffi.Pointer q, - ffi.Pointer g, - ffi.Pointer state, - double scale, ) { - return _ggml_gated_linear_attn( + return _ggml_new_graph( ctx, - k, - v, - q, - g, - state, - scale, ); } - late final _ggml_gated_linear_attnPtr = _lookup< + late final _ggml_new_graphPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Float)>>('ggml_gated_linear_attn'); - late final _ggml_gated_linear_attn = _ggml_gated_linear_attnPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - double)>(); + ffi.Pointer Function( + ffi.Pointer)>>('ggml_new_graph'); + late final _ggml_new_graph = _ggml_new_graphPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - ffi.Pointer ggml_rwkv_wkv7( + ffi.Pointer ggml_new_graph_custom( ffi.Pointer ctx, - ffi.Pointer r, - ffi.Pointer w, - ffi.Pointer k, - ffi.Pointer v, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer state, + int size, + bool grads, ) { - return _ggml_rwkv_wkv7( + return _ggml_new_graph_custom( ctx, - r, - w, - k, - v, - a, - b, - state, + size, + grads, ); } - late final _ggml_rwkv_wkv7Ptr = _lookup< + late final _ggml_new_graph_customPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_rwkv_wkv7'); - late final _ggml_rwkv_wkv7 = _ggml_rwkv_wkv7Ptr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, ffi.Size, + ffi.Bool)>>('ggml_new_graph_custom'); + late final _ggml_new_graph_custom = _ggml_new_graph_customPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, int, bool)>(); - ffi.Pointer ggml_solve_tri( + ffi.Pointer ggml_new_i32( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - bool left, - bool lower, - bool uni, + int value, ) { - return _ggml_solve_tri( + return _ggml_new_i32( ctx, - a, - b, - left, - lower, - uni, + value, ); } - late final _ggml_solve_triPtr = _lookup< + late final _ggml_new_i32Ptr = _lookup< ffi.NativeFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Bool, - ffi.Bool, - ffi.Bool)>>('ggml_solve_tri'); - late final _ggml_solve_tri = _ggml_solve_triPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - bool, - bool, - bool)>(); + ffi.Pointer, ffi.Int32)>>('ggml_new_i32'); + late final _ggml_new_i32 = _ggml_new_i32Ptr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - ffi.Pointer ggml_gated_delta_net( + ffi.Pointer ggml_new_tensor( ffi.Pointer ctx, - ffi.Pointer q, - ffi.Pointer k, - ffi.Pointer v, - ffi.Pointer g, - ffi.Pointer beta, - ffi.Pointer state, + ggml_type type, + int n_dims, + ffi.Pointer ne, ) { - return _ggml_gated_delta_net( + return _ggml_new_tensor( ctx, - q, - k, - v, - g, - beta, - state, + type.value, + n_dims, + ne, ); } - late final _ggml_gated_delta_netPtr = _lookup< + late final _ggml_new_tensorPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_gated_delta_net'); - late final _ggml_gated_delta_net = _ggml_gated_delta_netPtr.asFunction< + ffi.UnsignedInt, + ffi.Int, + ffi.Pointer)>>('ggml_new_tensor'); + late final _ggml_new_tensor = _ggml_new_tensorPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer, int, int, ffi.Pointer)>(); - ffi.Pointer ggml_map_custom1( + ffi.Pointer ggml_new_tensor_1d( ffi.Pointer ctx, - ffi.Pointer a, - ggml_custom1_op_t fun, - int n_tasks, - ffi.Pointer userdata, + ggml_type type, + int ne0, ) { - return _ggml_map_custom1( + return _ggml_new_tensor_1d( ctx, - a, - fun, - n_tasks, - userdata, + type.value, + ne0, ); } - late final _ggml_map_custom1Ptr = _lookup< + late final _ggml_new_tensor_1dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ggml_custom1_op_t, - ffi.Int, - ffi.Pointer)>>('ggml_map_custom1'); - late final _ggml_map_custom1 = _ggml_map_custom1Ptr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ggml_custom1_op_t, - int, - ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.UnsignedInt, ffi.Int64)>>('ggml_new_tensor_1d'); + late final _ggml_new_tensor_1d = _ggml_new_tensor_1dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int, int)>(); - ffi.Pointer ggml_map_custom1_inplace( + ffi.Pointer ggml_new_tensor_2d( ffi.Pointer ctx, - ffi.Pointer a, - ggml_custom1_op_t fun, - int n_tasks, - ffi.Pointer userdata, + ggml_type type, + int ne0, + int ne1, ) { - return _ggml_map_custom1_inplace( + return _ggml_new_tensor_2d( ctx, - a, - fun, - n_tasks, - userdata, + type.value, + ne0, + ne1, ); } - late final _ggml_map_custom1_inplacePtr = _lookup< + late final _ggml_new_tensor_2dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ggml_custom1_op_t, - ffi.Int, - ffi.Pointer)>>('ggml_map_custom1_inplace'); - late final _ggml_map_custom1_inplace = - _ggml_map_custom1_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ggml_custom1_op_t, - int, - ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.UnsignedInt, ffi.Int64, ffi.Int64)>>('ggml_new_tensor_2d'); + late final _ggml_new_tensor_2d = _ggml_new_tensor_2dPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, int, int, int)>(); - ffi.Pointer ggml_map_custom2( + ffi.Pointer ggml_new_tensor_3d( ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ggml_custom2_op_t fun, - int n_tasks, - ffi.Pointer userdata, + ggml_type type, + int ne0, + int ne1, + int ne2, ) { - return _ggml_map_custom2( + return _ggml_new_tensor_3d( ctx, - a, - b, - fun, - n_tasks, - userdata, + type.value, + ne0, + ne1, + ne2, ); } - late final _ggml_map_custom2Ptr = _lookup< + late final _ggml_new_tensor_3dPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_custom2_op_t, - ffi.Int, - ffi.Pointer)>>('ggml_map_custom2'); - late final _ggml_map_custom2 = _ggml_map_custom2Ptr.asFunction< + ffi.UnsignedInt, + ffi.Int64, + ffi.Int64, + ffi.Int64)>>('ggml_new_tensor_3d'); + late final _ggml_new_tensor_3d = _ggml_new_tensor_3dPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_custom2_op_t, - int, - ffi.Pointer)>(); + ffi.Pointer, int, int, int, int)>(); - ffi.Pointer ggml_map_custom2_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ggml_custom2_op_t fun, - int n_tasks, - ffi.Pointer userdata, - ) { - return _ggml_map_custom2_inplace( - ctx, - a, - b, - fun, - n_tasks, - userdata, - ); - } - - late final _ggml_map_custom2_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_custom2_op_t, - ffi.Int, - ffi.Pointer)>>('ggml_map_custom2_inplace'); - late final _ggml_map_custom2_inplace = - _ggml_map_custom2_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_custom2_op_t, - int, - ffi.Pointer)>(); - - ffi.Pointer ggml_map_custom3( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, - ggml_custom3_op_t fun, - int n_tasks, - ffi.Pointer userdata, - ) { - return _ggml_map_custom3( - ctx, - a, - b, - c, - fun, - n_tasks, - userdata, - ); - } - - late final _ggml_map_custom3Ptr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_custom3_op_t, - ffi.Int, - ffi.Pointer)>>('ggml_map_custom3'); - late final _ggml_map_custom3 = _ggml_map_custom3Ptr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_custom3_op_t, - int, - ffi.Pointer)>(); - - ffi.Pointer ggml_map_custom3_inplace( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, - ggml_custom3_op_t fun, - int n_tasks, - ffi.Pointer userdata, - ) { - return _ggml_map_custom3_inplace( - ctx, - a, - b, - c, - fun, - n_tasks, - userdata, - ); - } - - late final _ggml_map_custom3_inplacePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_custom3_op_t, - ffi.Int, - ffi.Pointer)>>('ggml_map_custom3_inplace'); - late final _ggml_map_custom3_inplace = - _ggml_map_custom3_inplacePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_custom3_op_t, - int, - ffi.Pointer)>(); - - ffi.Pointer ggml_custom_4d( + ffi.Pointer ggml_new_tensor_4d( ffi.Pointer ctx, ggml_type type, int ne0, int ne1, int ne2, int ne3, - ffi.Pointer> args, - int n_args, - ggml_custom_op_t fun, - int n_tasks, - ffi.Pointer userdata, ) { - return _ggml_custom_4d( + return _ggml_new_tensor_4d( ctx, type.value, ne0, ne1, ne2, ne3, - args, - n_args, - fun, - n_tasks, - userdata, ); } - late final _ggml_custom_4dPtr = _lookup< + late final _ggml_new_tensor_4dPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( ffi.Pointer, @@ -9082,3297 +8573,3991 @@ class LlamaBindings { ffi.Int64, ffi.Int64, ffi.Int64, - ffi.Int64, - ffi.Pointer>, - ffi.Int, - ggml_custom_op_t, - ffi.Int, - ffi.Pointer)>>('ggml_custom_4d'); - late final _ggml_custom_4d = _ggml_custom_4dPtr.asFunction< + ffi.Int64)>>('ggml_new_tensor_4d'); + late final _ggml_new_tensor_4d = _ggml_new_tensor_4dPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - int, - int, - int, - int, - int, - ffi.Pointer>, - int, - ggml_custom_op_t, - int, - ffi.Pointer)>(); + ffi.Pointer, int, int, int, int, int)>(); - ffi.Pointer ggml_custom_inplace( + ffi.Pointer ggml_norm( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer> args, - int n_args, - ggml_custom_op_t fun, - int n_tasks, - ffi.Pointer userdata, + double eps, ) { - return _ggml_custom_inplace( + return _ggml_norm( ctx, a, - args, - n_args, - fun, - n_tasks, - userdata, + eps, ); } - late final _ggml_custom_inplacePtr = _lookup< + late final _ggml_normPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer>, - ffi.Int, - ggml_custom_op_t, - ffi.Int, - ffi.Pointer)>>('ggml_custom_inplace'); - late final _ggml_custom_inplace = _ggml_custom_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_norm'); + late final _ggml_norm = _ggml_normPtr.asFunction< ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer>, - int, - ggml_custom_op_t, - int, - ffi.Pointer)>(); + ffi.Pointer, ffi.Pointer, double)>(); - ffi.Pointer ggml_cross_entropy_loss( + ffi.Pointer ggml_norm_inplace( ffi.Pointer ctx, ffi.Pointer a, - ffi.Pointer b, + double eps, ) { - return _ggml_cross_entropy_loss( + return _ggml_norm_inplace( ctx, a, - b, + eps, ); } - late final _ggml_cross_entropy_lossPtr = _lookup< + late final _ggml_norm_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_cross_entropy_loss'); - late final _ggml_cross_entropy_loss = _ggml_cross_entropy_lossPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_norm_inplace'); + late final _ggml_norm_inplace = _ggml_norm_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, double)>(); - ffi.Pointer ggml_cross_entropy_loss_back( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, + int ggml_nrows( + ffi.Pointer tensor, ) { - return _ggml_cross_entropy_loss_back( - ctx, - a, - b, - c, + return _ggml_nrows( + tensor, ); } - late final _ggml_cross_entropy_loss_backPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_cross_entropy_loss_back'); - late final _ggml_cross_entropy_loss_back = - _ggml_cross_entropy_loss_backPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + late final _ggml_nrowsPtr = + _lookup)>>( + 'ggml_nrows'); + late final _ggml_nrows = + _ggml_nrowsPtr.asFunction)>(); - ffi.Pointer ggml_opt_step_adamw( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer grad, - ffi.Pointer m, - ffi.Pointer v, - ffi.Pointer adamw_params, - ) { - return _ggml_opt_step_adamw( - ctx, - a, - grad, - m, - v, - adamw_params, + void ggml_numa_init( + ggml_numa_strategy numa, + ) { + return _ggml_numa_init( + numa.value, ); } - late final _ggml_opt_step_adamwPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_opt_step_adamw'); - late final _ggml_opt_step_adamw = _ggml_opt_step_adamwPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + late final _ggml_numa_initPtr = + _lookup>( + 'ggml_numa_init'); + late final _ggml_numa_init = + _ggml_numa_initPtr.asFunction(); - ffi.Pointer ggml_opt_step_sgd( - ffi.Pointer ctx, - ffi.Pointer a, - ffi.Pointer grad, - ffi.Pointer sgd_params, + ffi.Pointer ggml_op_desc( + ffi.Pointer t, ) { - return _ggml_opt_step_sgd( - ctx, - a, - grad, - sgd_params, + return _ggml_op_desc( + t, ); } - late final _ggml_opt_step_sgdPtr = _lookup< + late final _ggml_op_descPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_opt_step_sgd'); - late final _ggml_opt_step_sgd = _ggml_opt_step_sgdPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer)>>('ggml_op_desc'); + late final _ggml_op_desc = _ggml_op_descPtr + .asFunction Function(ffi.Pointer)>(); - ffi.Pointer ggml_build_forward_select( - ffi.Pointer cgraph, - ffi.Pointer> tensors, - int n_tensors, - int idx, + ffi.Pointer ggml_op_name( + ggml_op op, ) { - return _ggml_build_forward_select( - cgraph, - tensors, - n_tensors, - idx, + return _ggml_op_name( + op.value, ); } - late final _ggml_build_forward_selectPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer>, - ffi.Int, - ffi.Int)>>('ggml_build_forward_select'); - late final _ggml_build_forward_select = - _ggml_build_forward_selectPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer>, int, int)>(); + late final _ggml_op_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'ggml_op_name'); + late final _ggml_op_name = + _ggml_op_namePtr.asFunction Function(int)>(); - void ggml_build_forward_expand( - ffi.Pointer cgraph, - ffi.Pointer tensor, + ffi.Pointer ggml_op_symbol( + ggml_op op, ) { - return _ggml_build_forward_expand( - cgraph, - tensor, + return _ggml_op_symbol( + op.value, ); } - late final _ggml_build_forward_expandPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('ggml_build_forward_expand'); - late final _ggml_build_forward_expand = - _ggml_build_forward_expandPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + late final _ggml_op_symbolPtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'ggml_op_symbol'); + late final _ggml_op_symbol = + _ggml_op_symbolPtr.asFunction Function(int)>(); - void ggml_build_backward_expand( - ffi.Pointer ctx, - ffi.Pointer cgraph, - ffi.Pointer> grad_accs, + void ggml_opt_alloc( + ggml_opt_context_t opt_ctx, + bool backward, ) { - return _ggml_build_backward_expand( - ctx, - cgraph, - grad_accs, + return _ggml_opt_alloc( + opt_ctx, + backward, ); } - late final _ggml_build_backward_expandPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer>)>>( - 'ggml_build_backward_expand'); - late final _ggml_build_backward_expand = - _ggml_build_backward_expandPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer>)>(); + late final _ggml_opt_allocPtr = _lookup< + ffi.NativeFunction>( + 'ggml_opt_alloc'); + late final _ggml_opt_alloc = + _ggml_opt_allocPtr.asFunction(); - ffi.Pointer ggml_new_graph( - ffi.Pointer ctx, + ggml_opt_optimizer_type ggml_opt_context_optimizer_type( + ggml_opt_context_t arg0, ) { - return _ggml_new_graph( - ctx, - ); + return ggml_opt_optimizer_type.fromValue(_ggml_opt_context_optimizer_type( + arg0, + )); } - late final _ggml_new_graphPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('ggml_new_graph'); - late final _ggml_new_graph = _ggml_new_graphPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + late final _ggml_opt_context_optimizer_typePtr = + _lookup>( + 'ggml_opt_context_optimizer_type'); + late final _ggml_opt_context_optimizer_type = + _ggml_opt_context_optimizer_typePtr + .asFunction(); - ffi.Pointer ggml_new_graph_custom( - ffi.Pointer ctx, - int size, - bool grads, + ffi.Pointer ggml_opt_dataset_data( + ggml_opt_dataset_t dataset, ) { - return _ggml_new_graph_custom( - ctx, - size, - grads, + return _ggml_opt_dataset_data( + dataset, ); } - late final _ggml_new_graph_customPtr = _lookup< + late final _ggml_opt_dataset_dataPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, ffi.Size, - ffi.Bool)>>('ggml_new_graph_custom'); - late final _ggml_new_graph_custom = _ggml_new_graph_customPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int, bool)>(); + ffi.Pointer Function( + ggml_opt_dataset_t)>>('ggml_opt_dataset_data'); + late final _ggml_opt_dataset_data = _ggml_opt_dataset_dataPtr + .asFunction Function(ggml_opt_dataset_t)>(); - ffi.Pointer ggml_graph_dup( - ffi.Pointer ctx, - ffi.Pointer cgraph, - bool force_grads, + void ggml_opt_dataset_free( + ggml_opt_dataset_t dataset, ) { - return _ggml_graph_dup( - ctx, - cgraph, - force_grads, + return _ggml_opt_dataset_free( + dataset, ); } - late final _ggml_graph_dupPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Bool)>>('ggml_graph_dup'); - late final _ggml_graph_dup = _ggml_graph_dupPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer, bool)>(); + late final _ggml_opt_dataset_freePtr = + _lookup>( + 'ggml_opt_dataset_free'); + late final _ggml_opt_dataset_free = + _ggml_opt_dataset_freePtr.asFunction(); - void ggml_graph_cpy( - ffi.Pointer src, - ffi.Pointer dst, + void ggml_opt_dataset_get_batch( + ggml_opt_dataset_t dataset, + ffi.Pointer data_batch, + ffi.Pointer labels_batch, + int ibatch, ) { - return _ggml_graph_cpy( - src, - dst, + return _ggml_opt_dataset_get_batch( + dataset, + data_batch, + labels_batch, + ibatch, ); } - late final _ggml_graph_cpyPtr = _lookup< + late final _ggml_opt_dataset_get_batchPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('ggml_graph_cpy'); - late final _ggml_graph_cpy = _ggml_graph_cpyPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function( + ggml_opt_dataset_t, + ffi.Pointer, + ffi.Pointer, + ffi.Int64)>>('ggml_opt_dataset_get_batch'); + late final _ggml_opt_dataset_get_batch = + _ggml_opt_dataset_get_batchPtr.asFunction< + void Function(ggml_opt_dataset_t, ffi.Pointer, + ffi.Pointer, int)>(); - void ggml_graph_reset( - ffi.Pointer cgraph, + void ggml_opt_dataset_get_batch_host( + ggml_opt_dataset_t dataset, + ffi.Pointer data_batch, + int nb_data_batch, + ffi.Pointer labels_batch, + int ibatch, ) { - return _ggml_graph_reset( - cgraph, + return _ggml_opt_dataset_get_batch_host( + dataset, + data_batch, + nb_data_batch, + labels_batch, + ibatch, ); } - late final _ggml_graph_resetPtr = - _lookup)>>( - 'ggml_graph_reset'); - late final _ggml_graph_reset = _ggml_graph_resetPtr - .asFunction)>(); + late final _ggml_opt_dataset_get_batch_hostPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ggml_opt_dataset_t, + ffi.Pointer, + ffi.Size, + ffi.Pointer, + ffi.Int64)>>('ggml_opt_dataset_get_batch_host'); + late final _ggml_opt_dataset_get_batch_host = + _ggml_opt_dataset_get_batch_hostPtr.asFunction< + void Function(ggml_opt_dataset_t, ffi.Pointer, int, + ffi.Pointer, int)>(); - void ggml_graph_clear( - ffi.Pointer cgraph, + ggml_opt_dataset_t ggml_opt_dataset_init( + ggml_type type_data, + ggml_type type_label, + int ne_datapoint, + int ne_label, + int ndata, + int ndata_shard, ) { - return _ggml_graph_clear( - cgraph, + return _ggml_opt_dataset_init( + type_data.value, + type_label.value, + ne_datapoint, + ne_label, + ndata, + ndata_shard, ); } - late final _ggml_graph_clearPtr = - _lookup)>>( - 'ggml_graph_clear'); - late final _ggml_graph_clear = _ggml_graph_clearPtr - .asFunction)>(); + late final _ggml_opt_dataset_initPtr = _lookup< + ffi.NativeFunction< + ggml_opt_dataset_t Function( + ffi.UnsignedInt, + ffi.UnsignedInt, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Int64)>>('ggml_opt_dataset_init'); + late final _ggml_opt_dataset_init = _ggml_opt_dataset_initPtr + .asFunction(); - int ggml_graph_size( - ffi.Pointer cgraph, + ffi.Pointer ggml_opt_dataset_labels( + ggml_opt_dataset_t dataset, ) { - return _ggml_graph_size( - cgraph, + return _ggml_opt_dataset_labels( + dataset, ); } - late final _ggml_graph_sizePtr = - _lookup)>>( - 'ggml_graph_size'); - late final _ggml_graph_size = - _ggml_graph_sizePtr.asFunction)>(); + late final _ggml_opt_dataset_labelsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ggml_opt_dataset_t)>>('ggml_opt_dataset_labels'); + late final _ggml_opt_dataset_labels = _ggml_opt_dataset_labelsPtr + .asFunction Function(ggml_opt_dataset_t)>(); - ffi.Pointer ggml_graph_node( - ffi.Pointer cgraph, - int i, + int ggml_opt_dataset_ndata( + ggml_opt_dataset_t dataset, ) { - return _ggml_graph_node( - cgraph, - i, + return _ggml_opt_dataset_ndata( + dataset, ); } - late final _ggml_graph_nodePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int)>>('ggml_graph_node'); - late final _ggml_graph_node = _ggml_graph_nodePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + late final _ggml_opt_dataset_ndataPtr = + _lookup>( + 'ggml_opt_dataset_ndata'); + late final _ggml_opt_dataset_ndata = + _ggml_opt_dataset_ndataPtr.asFunction(); - ffi.Pointer> ggml_graph_nodes( - ffi.Pointer cgraph, + void ggml_opt_dataset_shuffle( + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + int idata, ) { - return _ggml_graph_nodes( - cgraph, + return _ggml_opt_dataset_shuffle( + opt_ctx, + dataset, + idata, ); } - late final _ggml_graph_nodesPtr = _lookup< + late final _ggml_opt_dataset_shufflePtr = _lookup< ffi.NativeFunction< - ffi.Pointer> Function( - ffi.Pointer)>>('ggml_graph_nodes'); - late final _ggml_graph_nodes = _ggml_graph_nodesPtr.asFunction< - ffi.Pointer> Function( - ffi.Pointer)>(); + ffi.Void Function(ggml_opt_context_t, ggml_opt_dataset_t, + ffi.Int64)>>('ggml_opt_dataset_shuffle'); + late final _ggml_opt_dataset_shuffle = _ggml_opt_dataset_shufflePtr + .asFunction(); - int ggml_graph_n_nodes( - ffi.Pointer cgraph, + ggml_opt_params ggml_opt_default_params( + ggml_backend_sched_t backend_sched, + ggml_opt_loss_type loss_type, ) { - return _ggml_graph_n_nodes( - cgraph, + return _ggml_opt_default_params( + backend_sched, + loss_type.value, ); } - late final _ggml_graph_n_nodesPtr = - _lookup)>>( - 'ggml_graph_n_nodes'); - late final _ggml_graph_n_nodes = _ggml_graph_n_nodesPtr - .asFunction)>(); + late final _ggml_opt_default_paramsPtr = _lookup< + ffi.NativeFunction< + ggml_opt_params Function(ggml_backend_sched_t, + ffi.UnsignedInt)>>('ggml_opt_default_params'); + late final _ggml_opt_default_params = _ggml_opt_default_paramsPtr + .asFunction(); - void ggml_graph_add_node( - ffi.Pointer cgraph, - ffi.Pointer tensor, + void ggml_opt_epoch( + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, + ggml_opt_result_t result_eval, + int idata_split, + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval, ) { - return _ggml_graph_add_node( - cgraph, - tensor, + return _ggml_opt_epoch( + opt_ctx, + dataset, + result_train, + result_eval, + idata_split, + callback_train, + callback_eval, ); } - late final _ggml_graph_add_nodePtr = _lookup< + late final _ggml_opt_epochPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('ggml_graph_add_node'); - late final _ggml_graph_add_node = _ggml_graph_add_nodePtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); - - int ggml_graph_overhead() { - return _ggml_graph_overhead(); - } - - late final _ggml_graph_overheadPtr = - _lookup>('ggml_graph_overhead'); - late final _ggml_graph_overhead = - _ggml_graph_overheadPtr.asFunction(); + ffi.Void Function( + ggml_opt_context_t, + ggml_opt_dataset_t, + ggml_opt_result_t, + ggml_opt_result_t, + ffi.Int64, + ggml_opt_epoch_callback, + ggml_opt_epoch_callback)>>('ggml_opt_epoch'); + late final _ggml_opt_epoch = _ggml_opt_epochPtr.asFunction< + void Function( + ggml_opt_context_t, + ggml_opt_dataset_t, + ggml_opt_result_t, + ggml_opt_result_t, + int, + ggml_opt_epoch_callback, + ggml_opt_epoch_callback)>(); - int ggml_graph_overhead_custom( - int size, - bool grads, + void ggml_opt_epoch_callback_progress_bar( + bool train, + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, + int ibatch, + int ibatch_max, + int t_start_us, ) { - return _ggml_graph_overhead_custom( - size, - grads, + return _ggml_opt_epoch_callback_progress_bar( + train, + opt_ctx, + dataset, + result, + ibatch, + ibatch_max, + t_start_us, ); } - late final _ggml_graph_overhead_customPtr = - _lookup>( - 'ggml_graph_overhead_custom'); - late final _ggml_graph_overhead_custom = - _ggml_graph_overhead_customPtr.asFunction(); + late final _ggml_opt_epoch_callback_progress_barPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ffi.Bool, + ggml_opt_context_t, + ggml_opt_dataset_t, + ggml_opt_result_t, + ffi.Int64, + ffi.Int64, + ffi.Int64)>>('ggml_opt_epoch_callback_progress_bar'); + late final _ggml_opt_epoch_callback_progress_bar = + _ggml_opt_epoch_callback_progress_barPtr.asFunction< + void Function(bool, ggml_opt_context_t, ggml_opt_dataset_t, + ggml_opt_result_t, int, int, int)>(); - ffi.Pointer ggml_graph_get_tensor( - ffi.Pointer cgraph, - ffi.Pointer name, + void ggml_opt_eval( + ggml_opt_context_t opt_ctx, + ggml_opt_result_t result, ) { - return _ggml_graph_get_tensor( - cgraph, - name, + return _ggml_opt_eval( + opt_ctx, + result, ); } - late final _ggml_graph_get_tensorPtr = _lookup< + late final _ggml_opt_evalPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_graph_get_tensor'); - late final _ggml_graph_get_tensor = _ggml_graph_get_tensorPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function( + ggml_opt_context_t, ggml_opt_result_t)>>('ggml_opt_eval'); + late final _ggml_opt_eval = _ggml_opt_evalPtr + .asFunction(); - ffi.Pointer ggml_graph_get_grad( - ffi.Pointer cgraph, - ffi.Pointer node, + void ggml_opt_fit( + ggml_backend_sched_t backend_sched, + ffi.Pointer ctx_compute, + ffi.Pointer inputs, + ffi.Pointer outputs, + ggml_opt_dataset_t dataset, + ggml_opt_loss_type loss_type, + ggml_opt_optimizer_type optimizer, + ggml_opt_get_optimizer_params get_opt_pars, + int nepoch, + int nbatch_logical, + double val_split, + bool silent, ) { - return _ggml_graph_get_grad( - cgraph, - node, + return _ggml_opt_fit( + backend_sched, + ctx_compute, + inputs, + outputs, + dataset, + loss_type.value, + optimizer.value, + get_opt_pars, + nepoch, + nbatch_logical, + val_split, + silent, ); } - late final _ggml_graph_get_gradPtr = _lookup< + late final _ggml_opt_fitPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_graph_get_grad'); - late final _ggml_graph_get_grad = _ggml_graph_get_gradPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function( + ggml_backend_sched_t, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ggml_opt_dataset_t, + ffi.UnsignedInt, + ffi.UnsignedInt, + ggml_opt_get_optimizer_params, + ffi.Int64, + ffi.Int64, + ffi.Float, + ffi.Bool)>>('ggml_opt_fit'); + late final _ggml_opt_fit = _ggml_opt_fitPtr.asFunction< + void Function( + ggml_backend_sched_t, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ggml_opt_dataset_t, + int, + int, + ggml_opt_get_optimizer_params, + int, + int, + double, + bool)>(); - ffi.Pointer ggml_graph_get_grad_acc( - ffi.Pointer cgraph, - ffi.Pointer node, + void ggml_opt_free( + ggml_opt_context_t opt_ctx, ) { - return _ggml_graph_get_grad_acc( - cgraph, - node, + return _ggml_opt_free( + opt_ctx, ); } - late final _ggml_graph_get_grad_accPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('ggml_graph_get_grad_acc'); - late final _ggml_graph_get_grad_acc = _ggml_graph_get_grad_accPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_opt_freePtr = + _lookup>( + 'ggml_opt_free'); + late final _ggml_opt_free = + _ggml_opt_freePtr.asFunction(); - void ggml_graph_print( - ffi.Pointer cgraph, + ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params( + ffi.Pointer userdata, ) { - return _ggml_graph_print( - cgraph, + return _ggml_opt_get_constant_optimizer_params( + userdata, ); } - late final _ggml_graph_printPtr = - _lookup)>>( - 'ggml_graph_print'); - late final _ggml_graph_print = _ggml_graph_printPtr - .asFunction)>(); + late final _ggml_opt_get_constant_optimizer_paramsPtr = _lookup< + ffi.NativeFunction< + ggml_opt_optimizer_params Function(ffi.Pointer)>>( + 'ggml_opt_get_constant_optimizer_params'); + late final _ggml_opt_get_constant_optimizer_params = + _ggml_opt_get_constant_optimizer_paramsPtr.asFunction< + ggml_opt_optimizer_params Function(ffi.Pointer)>(); - void ggml_graph_dump_dot( - ffi.Pointer gb, - ffi.Pointer cgraph, - ffi.Pointer filename, + ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params( + ffi.Pointer userdata, ) { - return _ggml_graph_dump_dot( - gb, - cgraph, - filename, + return _ggml_opt_get_default_optimizer_params( + userdata, ); } - late final _ggml_graph_dump_dotPtr = _lookup< + late final _ggml_opt_get_default_optimizer_paramsPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('ggml_graph_dump_dot'); - late final _ggml_graph_dump_dot = _ggml_graph_dump_dotPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer)>(); + ggml_opt_optimizer_params Function( + ffi.Pointer)>>('ggml_opt_get_default_optimizer_params'); + late final _ggml_opt_get_default_optimizer_params = + _ggml_opt_get_default_optimizer_paramsPtr.asFunction< + ggml_opt_optimizer_params Function(ffi.Pointer)>(); - void ggml_log_get( - ffi.Pointer log_callback, - ffi.Pointer> user_data, + ffi.Pointer ggml_opt_grad_acc( + ggml_opt_context_t opt_ctx, + ffi.Pointer node, ) { - return _ggml_log_get( - log_callback, - user_data, + return _ggml_opt_grad_acc( + opt_ctx, + node, ); } - late final _ggml_log_getPtr = _lookup< + late final _ggml_opt_grad_accPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer>)>>('ggml_log_get'); - late final _ggml_log_get = _ggml_log_getPtr.asFunction< - void Function(ffi.Pointer, - ffi.Pointer>)>(); + ffi.Pointer Function(ggml_opt_context_t, + ffi.Pointer)>>('ggml_opt_grad_acc'); + late final _ggml_opt_grad_acc = _ggml_opt_grad_accPtr.asFunction< + ffi.Pointer Function( + ggml_opt_context_t, ffi.Pointer)>(); - void ggml_log_set( - ggml_log_callback log_callback, - ffi.Pointer user_data, + ggml_opt_context_t ggml_opt_init( + ggml_opt_params params, ) { - return _ggml_log_set( - log_callback, - user_data, + return _ggml_opt_init( + params, ); } - late final _ggml_log_setPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ggml_log_callback, ffi.Pointer)>>('ggml_log_set'); - late final _ggml_log_set = _ggml_log_setPtr - .asFunction)>(); + late final _ggml_opt_initPtr = + _lookup>( + 'ggml_opt_init'); + late final _ggml_opt_init = _ggml_opt_initPtr + .asFunction(); - ffi.Pointer ggml_set_zero( - ffi.Pointer tensor, + ffi.Pointer ggml_opt_inputs( + ggml_opt_context_t opt_ctx, ) { - return _ggml_set_zero( - tensor, + return _ggml_opt_inputs( + opt_ctx, ); } - late final _ggml_set_zeroPtr = _lookup< + late final _ggml_opt_inputsPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( - ffi.Pointer)>>('ggml_set_zero'); - late final _ggml_set_zero = _ggml_set_zeroPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + ggml_opt_context_t)>>('ggml_opt_inputs'); + late final _ggml_opt_inputs = _ggml_opt_inputsPtr + .asFunction Function(ggml_opt_context_t)>(); - void ggml_quantize_init( - ggml_type type, + ffi.Pointer ggml_opt_labels( + ggml_opt_context_t opt_ctx, ) { - return _ggml_quantize_init( - type.value, + return _ggml_opt_labels( + opt_ctx, ); } - late final _ggml_quantize_initPtr = - _lookup>( - 'ggml_quantize_init'); - late final _ggml_quantize_init = - _ggml_quantize_initPtr.asFunction(); + late final _ggml_opt_labelsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ggml_opt_context_t)>>('ggml_opt_labels'); + late final _ggml_opt_labels = _ggml_opt_labelsPtr + .asFunction Function(ggml_opt_context_t)>(); - void ggml_quantize_free() { - return _ggml_quantize_free(); + ffi.Pointer ggml_opt_loss( + ggml_opt_context_t opt_ctx, + ) { + return _ggml_opt_loss( + opt_ctx, + ); } - late final _ggml_quantize_freePtr = - _lookup>('ggml_quantize_free'); - late final _ggml_quantize_free = - _ggml_quantize_freePtr.asFunction(); + late final _ggml_opt_lossPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ggml_opt_context_t)>>('ggml_opt_loss'); + late final _ggml_opt_loss = _ggml_opt_lossPtr + .asFunction Function(ggml_opt_context_t)>(); - bool ggml_quantize_requires_imatrix( - ggml_type type, + ffi.Pointer ggml_opt_ncorrect( + ggml_opt_context_t opt_ctx, ) { - return _ggml_quantize_requires_imatrix( - type.value, + return _ggml_opt_ncorrect( + opt_ctx, ); } - late final _ggml_quantize_requires_imatrixPtr = - _lookup>( - 'ggml_quantize_requires_imatrix'); - late final _ggml_quantize_requires_imatrix = - _ggml_quantize_requires_imatrixPtr.asFunction(); + late final _ggml_opt_ncorrectPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ggml_opt_context_t)>>('ggml_opt_ncorrect'); + late final _ggml_opt_ncorrect = _ggml_opt_ncorrectPtr + .asFunction Function(ggml_opt_context_t)>(); - int ggml_quantize_chunk( - ggml_type type, - ffi.Pointer src, - ffi.Pointer dst, - int start, - int nrows, - int n_per_row, - ffi.Pointer imatrix, + ffi.Pointer ggml_opt_optimizer_name( + ggml_opt_optimizer_type arg0, ) { - return _ggml_quantize_chunk( - type.value, - src, - dst, - start, - nrows, - n_per_row, - imatrix, + return _ggml_opt_optimizer_name( + arg0.value, ); } - late final _ggml_quantize_chunkPtr = _lookup< - ffi.NativeFunction< - ffi.Size Function( - ffi.UnsignedInt, - ffi.Pointer, - ffi.Pointer, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Pointer)>>('ggml_quantize_chunk'); - late final _ggml_quantize_chunk = _ggml_quantize_chunkPtr.asFunction< - int Function(int, ffi.Pointer, ffi.Pointer, int, int, - int, ffi.Pointer)>(); + late final _ggml_opt_optimizer_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'ggml_opt_optimizer_name'); + late final _ggml_opt_optimizer_name = _ggml_opt_optimizer_namePtr + .asFunction Function(int)>(); - ffi.Pointer ggml_get_type_traits( - ggml_type type, + ffi.Pointer ggml_opt_outputs( + ggml_opt_context_t opt_ctx, ) { - return _ggml_get_type_traits( - type.value, + return _ggml_opt_outputs( + opt_ctx, ); } - late final _ggml_get_type_traitsPtr = _lookup< + late final _ggml_opt_outputsPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.UnsignedInt)>>('ggml_get_type_traits'); - late final _ggml_get_type_traits = _ggml_get_type_traitsPtr - .asFunction Function(int)>(); + ffi.Pointer Function( + ggml_opt_context_t)>>('ggml_opt_outputs'); + late final _ggml_opt_outputs = _ggml_opt_outputsPtr + .asFunction Function(ggml_opt_context_t)>(); - ggml_threadpool_params ggml_threadpool_params_default( - int n_threads, + ffi.Pointer ggml_opt_pred( + ggml_opt_context_t opt_ctx, ) { - return _ggml_threadpool_params_default( - n_threads, + return _ggml_opt_pred( + opt_ctx, ); } - late final _ggml_threadpool_params_defaultPtr = - _lookup>( - 'ggml_threadpool_params_default'); - late final _ggml_threadpool_params_default = - _ggml_threadpool_params_defaultPtr - .asFunction(); + late final _ggml_opt_predPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ggml_opt_context_t)>>('ggml_opt_pred'); + late final _ggml_opt_pred = _ggml_opt_predPtr + .asFunction Function(ggml_opt_context_t)>(); - void ggml_threadpool_params_init( - ffi.Pointer p, - int n_threads, + void ggml_opt_prepare_alloc( + ggml_opt_context_t opt_ctx, + ffi.Pointer ctx_compute, + ffi.Pointer gf, + ffi.Pointer inputs, + ffi.Pointer outputs, ) { - return _ggml_threadpool_params_init( - p, - n_threads, + return _ggml_opt_prepare_alloc( + opt_ctx, + ctx_compute, + gf, + inputs, + outputs, ); } - late final _ggml_threadpool_params_initPtr = _lookup< + late final _ggml_opt_prepare_allocPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Int)>>('ggml_threadpool_params_init'); - late final _ggml_threadpool_params_init = _ggml_threadpool_params_initPtr - .asFunction, int)>(); + ffi.Void Function( + ggml_opt_context_t, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_opt_prepare_alloc'); + late final _ggml_opt_prepare_alloc = _ggml_opt_prepare_allocPtr.asFunction< + void Function( + ggml_opt_context_t, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - bool ggml_threadpool_params_match( - ffi.Pointer p0, - ffi.Pointer p1, + void ggml_opt_reset( + ggml_opt_context_t opt_ctx, + bool optimizer, ) { - return _ggml_threadpool_params_match( - p0, - p1, + return _ggml_opt_reset( + opt_ctx, + optimizer, ); } - late final _ggml_threadpool_params_matchPtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, - ffi.Pointer)>>( - 'ggml_threadpool_params_match'); - late final _ggml_threadpool_params_match = - _ggml_threadpool_params_matchPtr.asFunction< - bool Function(ffi.Pointer, - ffi.Pointer)>(); + late final _ggml_opt_resetPtr = _lookup< + ffi.NativeFunction>( + 'ggml_opt_reset'); + late final _ggml_opt_reset = + _ggml_opt_resetPtr.asFunction(); - ggml_tallocr ggml_tallocr_new( - ggml_backend_buffer_t buffer, + void ggml_opt_result_accuracy( + ggml_opt_result_t result, + ffi.Pointer accuracy, + ffi.Pointer unc, ) { - return _ggml_tallocr_new( - buffer, + return _ggml_opt_result_accuracy( + result, + accuracy, + unc, ); } - late final _ggml_tallocr_newPtr = - _lookup>( - 'ggml_tallocr_new'); - late final _ggml_tallocr_new = _ggml_tallocr_newPtr - .asFunction(); - - ggml_status ggml_tallocr_alloc( - ffi.Pointer talloc, - ffi.Pointer tensor, - ) { - return ggml_status.fromValue(_ggml_tallocr_alloc( - talloc, - tensor, - )); - } - - late final _ggml_tallocr_allocPtr = _lookup< + late final _ggml_opt_result_accuracyPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, - ffi.Pointer)>>('ggml_tallocr_alloc'); - late final _ggml_tallocr_alloc = _ggml_tallocr_allocPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ggml_opt_result_t, ffi.Pointer, + ffi.Pointer)>>('ggml_opt_result_accuracy'); + late final _ggml_opt_result_accuracy = + _ggml_opt_result_accuracyPtr.asFunction< + void Function(ggml_opt_result_t, ffi.Pointer, + ffi.Pointer)>(); - ggml_gallocr_t ggml_gallocr_new( - ggml_backend_buffer_type_t buft, + void ggml_opt_result_free( + ggml_opt_result_t result, ) { - return _ggml_gallocr_new( - buft, + return _ggml_opt_result_free( + result, ); } - late final _ggml_gallocr_newPtr = _lookup< - ffi - .NativeFunction>( - 'ggml_gallocr_new'); - late final _ggml_gallocr_new = _ggml_gallocr_newPtr - .asFunction(); + late final _ggml_opt_result_freePtr = + _lookup>( + 'ggml_opt_result_free'); + late final _ggml_opt_result_free = + _ggml_opt_result_freePtr.asFunction(); - ggml_gallocr_t ggml_gallocr_new_n( - ffi.Pointer bufts, - int n_bufs, - ) { - return _ggml_gallocr_new_n( - bufts, - n_bufs, - ); + ggml_opt_result_t ggml_opt_result_init() { + return _ggml_opt_result_init(); } - late final _ggml_gallocr_new_nPtr = _lookup< - ffi.NativeFunction< - ggml_gallocr_t Function(ffi.Pointer, - ffi.Int)>>('ggml_gallocr_new_n'); - late final _ggml_gallocr_new_n = _ggml_gallocr_new_nPtr.asFunction< - ggml_gallocr_t Function(ffi.Pointer, int)>(); + late final _ggml_opt_result_initPtr = + _lookup>( + 'ggml_opt_result_init'); + late final _ggml_opt_result_init = + _ggml_opt_result_initPtr.asFunction(); - void ggml_gallocr_free( - ggml_gallocr_t galloc, + void ggml_opt_result_loss( + ggml_opt_result_t result, + ffi.Pointer loss, + ffi.Pointer unc, ) { - return _ggml_gallocr_free( - galloc, + return _ggml_opt_result_loss( + result, + loss, + unc, ); } - late final _ggml_gallocr_freePtr = - _lookup>( - 'ggml_gallocr_free'); - late final _ggml_gallocr_free = - _ggml_gallocr_freePtr.asFunction(); + late final _ggml_opt_result_lossPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ggml_opt_result_t, ffi.Pointer, + ffi.Pointer)>>('ggml_opt_result_loss'); + late final _ggml_opt_result_loss = _ggml_opt_result_lossPtr.asFunction< + void Function(ggml_opt_result_t, ffi.Pointer, + ffi.Pointer)>(); - bool ggml_gallocr_reserve( - ggml_gallocr_t galloc, - ffi.Pointer graph, + void ggml_opt_result_ndata( + ggml_opt_result_t result, + ffi.Pointer ndata, ) { - return _ggml_gallocr_reserve( - galloc, - graph, + return _ggml_opt_result_ndata( + result, + ndata, ); } - late final _ggml_gallocr_reservePtr = _lookup< + late final _ggml_opt_result_ndataPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_gallocr_t, - ffi.Pointer)>>('ggml_gallocr_reserve'); - late final _ggml_gallocr_reserve = _ggml_gallocr_reservePtr - .asFunction)>(); + ffi.Void Function(ggml_opt_result_t, + ffi.Pointer)>>('ggml_opt_result_ndata'); + late final _ggml_opt_result_ndata = _ggml_opt_result_ndataPtr + .asFunction)>(); - void ggml_gallocr_reserve_n_size( - ggml_gallocr_t galloc, - ffi.Pointer graph, - ffi.Pointer node_buffer_ids, - ffi.Pointer leaf_buffer_ids, - ffi.Pointer sizes, + void ggml_opt_result_pred( + ggml_opt_result_t result, + ffi.Pointer pred, ) { - return _ggml_gallocr_reserve_n_size( - galloc, - graph, - node_buffer_ids, - leaf_buffer_ids, - sizes, + return _ggml_opt_result_pred( + result, + pred, ); } - late final _ggml_gallocr_reserve_n_sizePtr = _lookup< + late final _ggml_opt_result_predPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_gallocr_t, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_gallocr_reserve_n_size'); - late final _ggml_gallocr_reserve_n_size = - _ggml_gallocr_reserve_n_sizePtr.asFunction< - void Function( - ggml_gallocr_t, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Void Function(ggml_opt_result_t, + ffi.Pointer)>>('ggml_opt_result_pred'); + late final _ggml_opt_result_pred = _ggml_opt_result_predPtr + .asFunction)>(); - bool ggml_gallocr_reserve_n( - ggml_gallocr_t galloc, - ffi.Pointer graph, - ffi.Pointer node_buffer_ids, - ffi.Pointer leaf_buffer_ids, + void ggml_opt_result_reset( + ggml_opt_result_t result, ) { - return _ggml_gallocr_reserve_n( - galloc, - graph, - node_buffer_ids, - leaf_buffer_ids, + return _ggml_opt_result_reset( + result, ); } - late final _ggml_gallocr_reserve_nPtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function( - ggml_gallocr_t, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_gallocr_reserve_n'); - late final _ggml_gallocr_reserve_n = _ggml_gallocr_reserve_nPtr.asFunction< - bool Function(ggml_gallocr_t, ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + late final _ggml_opt_result_resetPtr = + _lookup>( + 'ggml_opt_result_reset'); + late final _ggml_opt_result_reset = + _ggml_opt_result_resetPtr.asFunction(); - bool ggml_gallocr_alloc_graph( - ggml_gallocr_t galloc, - ffi.Pointer graph, + bool ggml_opt_static_graphs( + ggml_opt_context_t opt_ctx, ) { - return _ggml_gallocr_alloc_graph( - galloc, - graph, + return _ggml_opt_static_graphs( + opt_ctx, ); } - late final _ggml_gallocr_alloc_graphPtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function(ggml_gallocr_t, - ffi.Pointer)>>('ggml_gallocr_alloc_graph'); - late final _ggml_gallocr_alloc_graph = _ggml_gallocr_alloc_graphPtr - .asFunction)>(); + late final _ggml_opt_static_graphsPtr = + _lookup>( + 'ggml_opt_static_graphs'); + late final _ggml_opt_static_graphs = _ggml_opt_static_graphsPtr + .asFunction(); - int ggml_gallocr_get_buffer_size( - ggml_gallocr_t galloc, - int buffer_id, + ffi.Pointer ggml_opt_step_adamw( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer grad, + ffi.Pointer m, + ffi.Pointer v, + ffi.Pointer adamw_params, ) { - return _ggml_gallocr_get_buffer_size( - galloc, - buffer_id, + return _ggml_opt_step_adamw( + ctx, + a, + grad, + m, + v, + adamw_params, ); } - late final _ggml_gallocr_get_buffer_sizePtr = - _lookup>( - 'ggml_gallocr_get_buffer_size'); - late final _ggml_gallocr_get_buffer_size = _ggml_gallocr_get_buffer_sizePtr - .asFunction(); + late final _ggml_opt_step_adamwPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_opt_step_adamw'); + late final _ggml_opt_step_adamw = _ggml_opt_step_adamwPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - int ggml_backend_alloc_ctx_tensors_from_buft_size( + ffi.Pointer ggml_opt_step_sgd( ffi.Pointer ctx, - ggml_backend_buffer_type_t buft, + ffi.Pointer a, + ffi.Pointer grad, + ffi.Pointer sgd_params, ) { - return _ggml_backend_alloc_ctx_tensors_from_buft_size( + return _ggml_opt_step_sgd( ctx, - buft, + a, + grad, + sgd_params, ); } - late final _ggml_backend_alloc_ctx_tensors_from_buft_sizePtr = _lookup< - ffi.NativeFunction< - ffi.Size Function( - ffi.Pointer, ggml_backend_buffer_type_t)>>( - 'ggml_backend_alloc_ctx_tensors_from_buft_size'); - late final _ggml_backend_alloc_ctx_tensors_from_buft_size = - _ggml_backend_alloc_ctx_tensors_from_buft_sizePtr.asFunction< - int Function( - ffi.Pointer, ggml_backend_buffer_type_t)>(); + late final _ggml_opt_step_sgdPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_opt_step_sgd'); + late final _ggml_opt_step_sgd = _ggml_opt_step_sgdPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - ffi.Pointer ggml_backend_alloc_ctx_tensors_from_buft( + ffi.Pointer ggml_out_prod( ffi.Pointer ctx, - ggml_backend_buffer_type_t buft, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_backend_alloc_ctx_tensors_from_buft( + return _ggml_out_prod( ctx, - buft, + a, + b, ); } - late final _ggml_backend_alloc_ctx_tensors_from_buftPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ggml_backend_buffer_type_t)>>( - 'ggml_backend_alloc_ctx_tensors_from_buft'); - late final _ggml_backend_alloc_ctx_tensors_from_buft = - _ggml_backend_alloc_ctx_tensors_from_buftPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ggml_backend_buffer_type_t)>(); + late final _ggml_out_prodPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_out_prod'); + late final _ggml_out_prod = _ggml_out_prodPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_backend_alloc_ctx_tensors( + ffi.Pointer ggml_pad( ffi.Pointer ctx, - ggml_backend_t backend, + ffi.Pointer a, + int p0, + int p1, + int p2, + int p3, ) { - return _ggml_backend_alloc_ctx_tensors( + return _ggml_pad( ctx, - backend, + a, + p0, + p1, + p2, + p3, ); } - late final _ggml_backend_alloc_ctx_tensorsPtr = _lookup< + late final _ggml_padPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ggml_backend_t)>>('ggml_backend_alloc_ctx_tensors'); - late final _ggml_backend_alloc_ctx_tensors = - _ggml_backend_alloc_ctx_tensorsPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ggml_backend_t)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_pad'); + late final _ggml_pad = _ggml_padPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); - ffi.Pointer ggml_backend_buft_name( - ggml_backend_buffer_type_t buft, + ffi.Pointer ggml_pad_circular( + ffi.Pointer ctx, + ffi.Pointer a, + int p0, + int p1, + int p2, + int p3, ) { - return _ggml_backend_buft_name( - buft, + return _ggml_pad_circular( + ctx, + a, + p0, + p1, + p2, + p3, ); } - late final _ggml_backend_buft_namePtr = _lookup< + late final _ggml_pad_circularPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_backend_buffer_type_t)>>('ggml_backend_buft_name'); - late final _ggml_backend_buft_name = _ggml_backend_buft_namePtr - .asFunction Function(ggml_backend_buffer_type_t)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_pad_circular'); + late final _ggml_pad_circular = _ggml_pad_circularPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); - ggml_backend_buffer_t ggml_backend_buft_alloc_buffer( - ggml_backend_buffer_type_t buft, - int size, + ffi.Pointer ggml_pad_ext( + ffi.Pointer ctx, + ffi.Pointer a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3, ) { - return _ggml_backend_buft_alloc_buffer( - buft, - size, + return _ggml_pad_ext( + ctx, + a, + lp0, + rp0, + lp1, + rp1, + lp2, + rp2, + lp3, + rp3, ); } - late final _ggml_backend_buft_alloc_bufferPtr = _lookup< + late final _ggml_pad_extPtr = _lookup< ffi.NativeFunction< - ggml_backend_buffer_t Function(ggml_backend_buffer_type_t, - ffi.Size)>>('ggml_backend_buft_alloc_buffer'); - late final _ggml_backend_buft_alloc_buffer = - _ggml_backend_buft_alloc_bufferPtr.asFunction< - ggml_backend_buffer_t Function(ggml_backend_buffer_type_t, int)>(); - - int ggml_backend_buft_get_alignment( - ggml_backend_buffer_type_t buft, - ) { - return _ggml_backend_buft_get_alignment( - buft, - ); - } - - late final _ggml_backend_buft_get_alignmentPtr = _lookup< - ffi.NativeFunction>( - 'ggml_backend_buft_get_alignment'); - late final _ggml_backend_buft_get_alignment = - _ggml_backend_buft_get_alignmentPtr - .asFunction(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_pad_ext'); + late final _ggml_pad_ext = _ggml_pad_extPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int, int, int, int, int)>(); - int ggml_backend_buft_get_max_size( - ggml_backend_buffer_type_t buft, + ffi.Pointer ggml_pad_ext_circular( + ffi.Pointer ctx, + ffi.Pointer a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3, ) { - return _ggml_backend_buft_get_max_size( - buft, + return _ggml_pad_ext_circular( + ctx, + a, + lp0, + rp0, + lp1, + rp1, + lp2, + rp2, + lp3, + rp3, ); } - late final _ggml_backend_buft_get_max_sizePtr = _lookup< - ffi.NativeFunction>( - 'ggml_backend_buft_get_max_size'); - late final _ggml_backend_buft_get_max_size = - _ggml_backend_buft_get_max_sizePtr - .asFunction(); + late final _ggml_pad_ext_circularPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_pad_ext_circular'); + late final _ggml_pad_ext_circular = _ggml_pad_ext_circularPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int, int, int, int, int)>(); - int ggml_backend_buft_get_alloc_size( - ggml_backend_buffer_type_t buft, - ffi.Pointer tensor, + ffi.Pointer ggml_pad_reflect_1d( + ffi.Pointer ctx, + ffi.Pointer a, + int p0, + int p1, ) { - return _ggml_backend_buft_get_alloc_size( - buft, - tensor, + return _ggml_pad_reflect_1d( + ctx, + a, + p0, + p1, ); } - late final _ggml_backend_buft_get_alloc_sizePtr = _lookup< + late final _ggml_pad_reflect_1dPtr = _lookup< ffi.NativeFunction< - ffi.Size Function(ggml_backend_buffer_type_t, - ffi.Pointer)>>('ggml_backend_buft_get_alloc_size'); - late final _ggml_backend_buft_get_alloc_size = - _ggml_backend_buft_get_alloc_sizePtr.asFunction< - int Function(ggml_backend_buffer_type_t, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int)>>('ggml_pad_reflect_1d'); + late final _ggml_pad_reflect_1d = _ggml_pad_reflect_1dPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - bool ggml_backend_buft_is_host( - ggml_backend_buffer_type_t buft, + ffi.Pointer ggml_permute( + ffi.Pointer ctx, + ffi.Pointer a, + int axis0, + int axis1, + int axis2, + int axis3, ) { - return _ggml_backend_buft_is_host( - buft, + return _ggml_permute( + ctx, + a, + axis0, + axis1, + axis2, + axis3, ); } - late final _ggml_backend_buft_is_hostPtr = _lookup< - ffi.NativeFunction>( - 'ggml_backend_buft_is_host'); - late final _ggml_backend_buft_is_host = _ggml_backend_buft_is_hostPtr - .asFunction(); + late final _ggml_permutePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_permute'); + late final _ggml_permute = _ggml_permutePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); - ggml_backend_dev_t ggml_backend_buft_get_device( - ggml_backend_buffer_type_t buft, + ffi.Pointer ggml_pool_1d( + ffi.Pointer ctx, + ffi.Pointer a, + ggml_op_pool op, + int k0, + int s0, + int p0, ) { - return _ggml_backend_buft_get_device( - buft, + return _ggml_pool_1d( + ctx, + a, + op.value, + k0, + s0, + p0, ); } - late final _ggml_backend_buft_get_devicePtr = _lookup< + late final _ggml_pool_1dPtr = _lookup< ffi.NativeFunction< - ggml_backend_dev_t Function( - ggml_backend_buffer_type_t)>>('ggml_backend_buft_get_device'); - late final _ggml_backend_buft_get_device = _ggml_backend_buft_get_devicePtr - .asFunction(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.UnsignedInt, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_pool_1d'); + late final _ggml_pool_1d = _ggml_pool_1dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); - ffi.Pointer ggml_backend_buffer_name( - ggml_backend_buffer_t buffer, + ffi.Pointer ggml_pool_2d( + ffi.Pointer ctx, + ffi.Pointer a, + ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + double p0, + double p1, ) { - return _ggml_backend_buffer_name( - buffer, + return _ggml_pool_2d( + ctx, + a, + op.value, + k0, + k1, + s0, + s1, + p0, + p1, ); } - late final _ggml_backend_buffer_namePtr = _lookup< + late final _ggml_pool_2dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_backend_buffer_t)>>('ggml_backend_buffer_name'); - late final _ggml_backend_buffer_name = _ggml_backend_buffer_namePtr - .asFunction Function(ggml_backend_buffer_t)>(); - - void ggml_backend_buffer_free( - ggml_backend_buffer_t buffer, - ) { - return _ggml_backend_buffer_free( - buffer, - ); - } - - late final _ggml_backend_buffer_freePtr = - _lookup>( - 'ggml_backend_buffer_free'); - late final _ggml_backend_buffer_free = _ggml_backend_buffer_freePtr - .asFunction(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.UnsignedInt, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float)>>('ggml_pool_2d'); + late final _ggml_pool_2d = _ggml_pool_2dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int, int, double, double)>(); - ffi.Pointer ggml_backend_buffer_get_base( - ggml_backend_buffer_t buffer, + ffi.Pointer ggml_pool_2d_back( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer af, + ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + double p0, + double p1, ) { - return _ggml_backend_buffer_get_base( - buffer, + return _ggml_pool_2d_back( + ctx, + a, + af, + op.value, + k0, + k1, + s0, + s1, + p0, + p1, ); } - late final _ggml_backend_buffer_get_basePtr = _lookup< + late final _ggml_pool_2d_backPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_backend_buffer_t)>>('ggml_backend_buffer_get_base'); - late final _ggml_backend_buffer_get_base = _ggml_backend_buffer_get_basePtr - .asFunction Function(ggml_backend_buffer_t)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.UnsignedInt, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float)>>('ggml_pool_2d_back'); + late final _ggml_pool_2d_back = _ggml_pool_2d_backPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int, + int, + double, + double)>(); - int ggml_backend_buffer_get_size( - ggml_backend_buffer_t buffer, + void ggml_print_object( + ffi.Pointer obj, ) { - return _ggml_backend_buffer_get_size( - buffer, + return _ggml_print_object( + obj, ); } - late final _ggml_backend_buffer_get_sizePtr = - _lookup>( - 'ggml_backend_buffer_get_size'); - late final _ggml_backend_buffer_get_size = _ggml_backend_buffer_get_sizePtr - .asFunction(); + late final _ggml_print_objectPtr = + _lookup)>>( + 'ggml_print_object'); + late final _ggml_print_object = _ggml_print_objectPtr + .asFunction)>(); - ggml_status ggml_backend_buffer_init_tensor( - ggml_backend_buffer_t buffer, - ffi.Pointer tensor, + void ggml_print_objects( + ffi.Pointer ctx, ) { - return ggml_status.fromValue(_ggml_backend_buffer_init_tensor( - buffer, - tensor, - )); + return _ggml_print_objects( + ctx, + ); } - late final _ggml_backend_buffer_init_tensorPtr = _lookup< - ffi.NativeFunction< - ffi.Int Function(ggml_backend_buffer_t, - ffi.Pointer)>>('ggml_backend_buffer_init_tensor'); - late final _ggml_backend_buffer_init_tensor = - _ggml_backend_buffer_init_tensorPtr.asFunction< - int Function(ggml_backend_buffer_t, ffi.Pointer)>(); + late final _ggml_print_objectsPtr = + _lookup)>>( + 'ggml_print_objects'); + late final _ggml_print_objects = _ggml_print_objectsPtr + .asFunction)>(); - int ggml_backend_buffer_get_alignment( - ggml_backend_buffer_t buffer, + int ggml_quantize_chunk( + ggml_type type, + ffi.Pointer src, + ffi.Pointer dst, + int start, + int nrows, + int n_per_row, + ffi.Pointer imatrix, ) { - return _ggml_backend_buffer_get_alignment( - buffer, + return _ggml_quantize_chunk( + type.value, + src, + dst, + start, + nrows, + n_per_row, + imatrix, ); } - late final _ggml_backend_buffer_get_alignmentPtr = - _lookup>( - 'ggml_backend_buffer_get_alignment'); - late final _ggml_backend_buffer_get_alignment = - _ggml_backend_buffer_get_alignmentPtr - .asFunction(); + late final _ggml_quantize_chunkPtr = _lookup< + ffi.NativeFunction< + ffi.Size Function( + ffi.UnsignedInt, + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Pointer)>>('ggml_quantize_chunk'); + late final _ggml_quantize_chunk = _ggml_quantize_chunkPtr.asFunction< + int Function(int, ffi.Pointer, ffi.Pointer, int, int, + int, ffi.Pointer)>(); - int ggml_backend_buffer_get_max_size( - ggml_backend_buffer_t buffer, - ) { - return _ggml_backend_buffer_get_max_size( - buffer, - ); + void ggml_quantize_free() { + return _ggml_quantize_free(); } - late final _ggml_backend_buffer_get_max_sizePtr = - _lookup>( - 'ggml_backend_buffer_get_max_size'); - late final _ggml_backend_buffer_get_max_size = - _ggml_backend_buffer_get_max_sizePtr - .asFunction(); + late final _ggml_quantize_freePtr = + _lookup>('ggml_quantize_free'); + late final _ggml_quantize_free = + _ggml_quantize_freePtr.asFunction(); - int ggml_backend_buffer_get_alloc_size( - ggml_backend_buffer_t buffer, - ffi.Pointer tensor, + void ggml_quantize_init( + ggml_type type, ) { - return _ggml_backend_buffer_get_alloc_size( - buffer, - tensor, + return _ggml_quantize_init( + type.value, ); } - late final _ggml_backend_buffer_get_alloc_sizePtr = _lookup< - ffi.NativeFunction< - ffi.Size Function(ggml_backend_buffer_t, - ffi.Pointer)>>('ggml_backend_buffer_get_alloc_size'); - late final _ggml_backend_buffer_get_alloc_size = - _ggml_backend_buffer_get_alloc_sizePtr.asFunction< - int Function(ggml_backend_buffer_t, ffi.Pointer)>(); + late final _ggml_quantize_initPtr = + _lookup>( + 'ggml_quantize_init'); + late final _ggml_quantize_init = + _ggml_quantize_initPtr.asFunction(); - void ggml_backend_buffer_clear( - ggml_backend_buffer_t buffer, - int value, + bool ggml_quantize_requires_imatrix( + ggml_type type, ) { - return _ggml_backend_buffer_clear( - buffer, - value, + return _ggml_quantize_requires_imatrix( + type.value, ); } - late final _ggml_backend_buffer_clearPtr = _lookup< - ffi - .NativeFunction>( - 'ggml_backend_buffer_clear'); - late final _ggml_backend_buffer_clear = _ggml_backend_buffer_clearPtr - .asFunction(); + late final _ggml_quantize_requires_imatrixPtr = + _lookup>( + 'ggml_quantize_requires_imatrix'); + late final _ggml_quantize_requires_imatrix = + _ggml_quantize_requires_imatrixPtr.asFunction(); - bool ggml_backend_buffer_is_host( - ggml_backend_buffer_t buffer, + ffi.Pointer ggml_reglu( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_buffer_is_host( - buffer, + return _ggml_reglu( + ctx, + a, ); } - late final _ggml_backend_buffer_is_hostPtr = - _lookup>( - 'ggml_backend_buffer_is_host'); - late final _ggml_backend_buffer_is_host = _ggml_backend_buffer_is_hostPtr - .asFunction(); + late final _ggml_regluPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_reglu'); + late final _ggml_reglu = _ggml_regluPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_buffer_set_usage( - ggml_backend_buffer_t buffer, - ggml_backend_buffer_usage usage, + ffi.Pointer ggml_reglu_split( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_backend_buffer_set_usage( - buffer, - usage.value, + return _ggml_reglu_split( + ctx, + a, + b, ); } - late final _ggml_backend_buffer_set_usagePtr = _lookup< + late final _ggml_reglu_splitPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_buffer_t, - ffi.UnsignedInt)>>('ggml_backend_buffer_set_usage'); - late final _ggml_backend_buffer_set_usage = _ggml_backend_buffer_set_usagePtr - .asFunction(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_reglu_split'); + late final _ggml_reglu_split = _ggml_reglu_splitPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_buffer_usage ggml_backend_buffer_get_usage( - ggml_backend_buffer_t buffer, + ffi.Pointer ggml_reglu_swapped( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return ggml_backend_buffer_usage.fromValue(_ggml_backend_buffer_get_usage( - buffer, - )); + return _ggml_reglu_swapped( + ctx, + a, + ); } - late final _ggml_backend_buffer_get_usagePtr = _lookup< - ffi.NativeFunction>( - 'ggml_backend_buffer_get_usage'); - late final _ggml_backend_buffer_get_usage = _ggml_backend_buffer_get_usagePtr - .asFunction(); + late final _ggml_reglu_swappedPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_reglu_swapped'); + late final _ggml_reglu_swapped = _ggml_reglu_swappedPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_buffer_type_t ggml_backend_buffer_get_type( - ggml_backend_buffer_t buffer, + ffi.Pointer ggml_relu( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_buffer_get_type( - buffer, + return _ggml_relu( + ctx, + a, ); } - late final _ggml_backend_buffer_get_typePtr = _lookup< + late final _ggml_reluPtr = _lookup< ffi.NativeFunction< - ggml_backend_buffer_type_t Function( - ggml_backend_buffer_t)>>('ggml_backend_buffer_get_type'); - late final _ggml_backend_buffer_get_type = _ggml_backend_buffer_get_typePtr - .asFunction(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_relu'); + late final _ggml_relu = _ggml_reluPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_buffer_reset( - ggml_backend_buffer_t buffer, + ffi.Pointer ggml_relu_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_buffer_reset( - buffer, + return _ggml_relu_inplace( + ctx, + a, ); } - late final _ggml_backend_buffer_resetPtr = - _lookup>( - 'ggml_backend_buffer_reset'); - late final _ggml_backend_buffer_reset = _ggml_backend_buffer_resetPtr - .asFunction(); + late final _ggml_relu_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_relu_inplace'); + late final _ggml_relu_inplace = _ggml_relu_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_tensor_copy( - ffi.Pointer src, - ffi.Pointer dst, + ffi.Pointer ggml_repeat( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_backend_tensor_copy( - src, - dst, + return _ggml_repeat( + ctx, + a, + b, ); } - late final _ggml_backend_tensor_copyPtr = _lookup< + late final _ggml_repeatPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('ggml_backend_tensor_copy'); - late final _ggml_backend_tensor_copy = - _ggml_backend_tensor_copyPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_repeat'); + late final _ggml_repeat = _ggml_repeatPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ggml_guid_t ggml_backend_guid( - ggml_backend_t backend, + ffi.Pointer ggml_repeat_4d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int ne1, + int ne2, + int ne3, ) { - return _ggml_backend_guid( - backend, + return _ggml_repeat_4d( + ctx, + a, + ne0, + ne1, + ne2, + ne3, ); } - late final _ggml_backend_guidPtr = - _lookup>( - 'ggml_backend_guid'); - late final _ggml_backend_guid = - _ggml_backend_guidPtr.asFunction(); + late final _ggml_repeat_4dPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Int64)>>('ggml_repeat_4d'); + late final _ggml_repeat_4d = _ggml_repeat_4dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); - ffi.Pointer ggml_backend_name( - ggml_backend_t backend, + ffi.Pointer ggml_repeat_back( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_backend_name( - backend, + return _ggml_repeat_back( + ctx, + a, + b, ); } - late final _ggml_backend_namePtr = _lookup< - ffi.NativeFunction Function(ggml_backend_t)>>( - 'ggml_backend_name'); - late final _ggml_backend_name = _ggml_backend_namePtr - .asFunction Function(ggml_backend_t)>(); + late final _ggml_repeat_backPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_repeat_back'); + late final _ggml_repeat_back = _ggml_repeat_backPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_free( - ggml_backend_t backend, + void ggml_reset( + ffi.Pointer ctx, ) { - return _ggml_backend_free( - backend, + return _ggml_reset( + ctx, ); } - late final _ggml_backend_freePtr = - _lookup>( - 'ggml_backend_free'); - late final _ggml_backend_free = - _ggml_backend_freePtr.asFunction(); + late final _ggml_resetPtr = + _lookup)>>( + 'ggml_reset'); + late final _ggml_reset = + _ggml_resetPtr.asFunction)>(); - ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type( - ggml_backend_t backend, + ffi.Pointer ggml_reshape( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_backend_get_default_buffer_type( - backend, + return _ggml_reshape( + ctx, + a, + b, ); } - late final _ggml_backend_get_default_buffer_typePtr = _lookup< - ffi - .NativeFunction>( - 'ggml_backend_get_default_buffer_type'); - late final _ggml_backend_get_default_buffer_type = - _ggml_backend_get_default_buffer_typePtr - .asFunction(); + late final _ggml_reshapePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_reshape'); + late final _ggml_reshape = _ggml_reshapePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_buffer_t ggml_backend_alloc_buffer( - ggml_backend_t backend, - int size, + ffi.Pointer ggml_reshape_1d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, ) { - return _ggml_backend_alloc_buffer( - backend, - size, + return _ggml_reshape_1d( + ctx, + a, + ne0, ); } - late final _ggml_backend_alloc_bufferPtr = _lookup< + late final _ggml_reshape_1dPtr = _lookup< ffi.NativeFunction< - ggml_backend_buffer_t Function( - ggml_backend_t, ffi.Size)>>('ggml_backend_alloc_buffer'); - late final _ggml_backend_alloc_buffer = _ggml_backend_alloc_bufferPtr - .asFunction(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Int64)>>('ggml_reshape_1d'); + late final _ggml_reshape_1d = _ggml_reshape_1dPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - int ggml_backend_get_alignment( - ggml_backend_t backend, + ffi.Pointer ggml_reshape_2d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int ne1, ) { - return _ggml_backend_get_alignment( - backend, + return _ggml_reshape_2d( + ctx, + a, + ne0, + ne1, ); } - late final _ggml_backend_get_alignmentPtr = - _lookup>( - 'ggml_backend_get_alignment'); - late final _ggml_backend_get_alignment = - _ggml_backend_get_alignmentPtr.asFunction(); + late final _ggml_reshape_2dPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64)>>('ggml_reshape_2d'); + late final _ggml_reshape_2d = _ggml_reshape_2dPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - int ggml_backend_get_max_size( - ggml_backend_t backend, + ffi.Pointer ggml_reshape_3d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int ne1, + int ne2, ) { - return _ggml_backend_get_max_size( - backend, + return _ggml_reshape_3d( + ctx, + a, + ne0, + ne1, + ne2, ); } - late final _ggml_backend_get_max_sizePtr = - _lookup>( - 'ggml_backend_get_max_size'); - late final _ggml_backend_get_max_size = - _ggml_backend_get_max_sizePtr.asFunction(); + late final _ggml_reshape_3dPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64, + ffi.Int64)>>('ggml_reshape_3d'); + late final _ggml_reshape_3d = _ggml_reshape_3dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int)>(); - void ggml_backend_tensor_set_async( - ggml_backend_t backend, - ffi.Pointer tensor, - ffi.Pointer data, - int offset, - int size, + ffi.Pointer ggml_reshape_4d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int ne1, + int ne2, + int ne3, ) { - return _ggml_backend_tensor_set_async( - backend, - tensor, - data, - offset, - size, + return _ggml_reshape_4d( + ctx, + a, + ne0, + ne1, + ne2, + ne3, ); } - late final _ggml_backend_tensor_set_asyncPtr = _lookup< + late final _ggml_reshape_4dPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_backend_t, + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size)>>('ggml_backend_tensor_set_async'); - late final _ggml_backend_tensor_set_async = - _ggml_backend_tensor_set_asyncPtr.asFunction< - void Function(ggml_backend_t, ffi.Pointer, - ffi.Pointer, int, int)>(); + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Int64)>>('ggml_reshape_4d'); + late final _ggml_reshape_4d = _ggml_reshape_4dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); - void ggml_backend_tensor_get_async( - ggml_backend_t backend, - ffi.Pointer tensor, - ffi.Pointer data, - int offset, - int size, + ffi.Pointer ggml_rms_norm( + ffi.Pointer ctx, + ffi.Pointer a, + double eps, ) { - return _ggml_backend_tensor_get_async( - backend, - tensor, - data, - offset, - size, + return _ggml_rms_norm( + ctx, + a, + eps, ); } - late final _ggml_backend_tensor_get_asyncPtr = _lookup< + late final _ggml_rms_normPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_backend_t, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size)>>('ggml_backend_tensor_get_async'); - late final _ggml_backend_tensor_get_async = - _ggml_backend_tensor_get_asyncPtr.asFunction< - void Function(ggml_backend_t, ffi.Pointer, - ffi.Pointer, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_rms_norm'); + late final _ggml_rms_norm = _ggml_rms_normPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, double)>(); - void ggml_backend_tensor_set_2d_async( - ggml_backend_t backend, - ffi.Pointer tensor, - ffi.Pointer data, - int offset, - int size, - int n_copies, - int stride_tensor, - int stride_data, + ffi.Pointer ggml_rms_norm_back( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + double eps, ) { - return _ggml_backend_tensor_set_2d_async( - backend, - tensor, - data, - offset, - size, - n_copies, - stride_tensor, - stride_data, + return _ggml_rms_norm_back( + ctx, + a, + b, + eps, ); } - late final _ggml_backend_tensor_set_2d_asyncPtr = _lookup< + late final _ggml_rms_norm_backPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_backend_t, + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_backend_tensor_set_2d_async'); - late final _ggml_backend_tensor_set_2d_async = - _ggml_backend_tensor_set_2d_asyncPtr.asFunction< - void Function(ggml_backend_t, ffi.Pointer, - ffi.Pointer, int, int, int, int, int)>(); + ffi.Pointer, + ffi.Float)>>('ggml_rms_norm_back'); + late final _ggml_rms_norm_back = _ggml_rms_norm_backPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, double)>(); - void ggml_backend_tensor_get_2d_async( - ggml_backend_t backend, - ffi.Pointer tensor, - ffi.Pointer data, - int offset, - int size, - int n_copies, - int stride_tensor, - int stride_data, + ffi.Pointer ggml_rms_norm_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + double eps, ) { - return _ggml_backend_tensor_get_2d_async( - backend, - tensor, - data, - offset, - size, - n_copies, - stride_tensor, - stride_data, + return _ggml_rms_norm_inplace( + ctx, + a, + eps, ); } - late final _ggml_backend_tensor_get_2d_asyncPtr = _lookup< + late final _ggml_rms_norm_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_backend_t, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_backend_tensor_get_2d_async'); - late final _ggml_backend_tensor_get_2d_async = - _ggml_backend_tensor_get_2d_asyncPtr.asFunction< - void Function(ggml_backend_t, ffi.Pointer, - ffi.Pointer, int, int, int, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_rms_norm_inplace'); + late final _ggml_rms_norm_inplace = _ggml_rms_norm_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, double)>(); - void ggml_backend_tensor_set( - ffi.Pointer tensor, - ffi.Pointer data, - int offset, - int size, + ffi.Pointer ggml_roll( + ffi.Pointer ctx, + ffi.Pointer a, + int shift0, + int shift1, + int shift2, + int shift3, ) { - return _ggml_backend_tensor_set( - tensor, - data, - offset, - size, + return _ggml_roll( + ctx, + a, + shift0, + shift1, + shift2, + shift3, ); } - late final _ggml_backend_tensor_setPtr = _lookup< + late final _ggml_rollPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Size, ffi.Size)>>('ggml_backend_tensor_set'); - late final _ggml_backend_tensor_set = _ggml_backend_tensor_setPtr.asFunction< - void Function( - ffi.Pointer, ffi.Pointer, int, int)>(); - - void ggml_backend_tensor_get( - ffi.Pointer tensor, - ffi.Pointer data, - int offset, - int size, + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_roll'); + late final _ggml_roll = _ggml_rollPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); + + ffi.Pointer ggml_rope( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int n_dims, + int mode, ) { - return _ggml_backend_tensor_get( - tensor, - data, - offset, - size, + return _ggml_rope( + ctx, + a, + b, + n_dims, + mode, ); } - late final _ggml_backend_tensor_getPtr = _lookup< + late final _ggml_ropePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Size, ffi.Size)>>('ggml_backend_tensor_get'); - late final _ggml_backend_tensor_get = _ggml_backend_tensor_getPtr.asFunction< - void Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int)>>('ggml_rope'); + late final _ggml_rope = _ggml_ropePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int, int)>(); - void ggml_backend_tensor_set_2d( - ffi.Pointer tensor, - ffi.Pointer data, - int offset, - int size, - int n_copies, - int stride_tensor, - int stride_data, + @Deprecated('use ggml_rope_ext instead') + ffi.Pointer ggml_rope_custom( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int n_dims, + int mode, + int n_ctx_orig, + double freq_base, + double freq_scale, + double ext_factor, + double attn_factor, + double beta_fast, + double beta_slow, ) { - return _ggml_backend_tensor_set_2d( - tensor, - data, - offset, - size, - n_copies, - stride_tensor, - stride_data, + return _ggml_rope_custom( + ctx, + a, + b, + n_dims, + mode, + n_ctx_orig, + freq_base, + freq_scale, + ext_factor, + attn_factor, + beta_fast, + beta_slow, ); } - late final _ggml_backend_tensor_set_2dPtr = _lookup< + late final _ggml_rope_customPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_backend_tensor_set_2d'); - late final _ggml_backend_tensor_set_2d = - _ggml_backend_tensor_set_2dPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int, - int, int, int, int)>(); + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_rope_custom'); + late final _ggml_rope_custom = _ggml_rope_customPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + double, + double, + double, + double, + double, + double)>(); - void ggml_backend_tensor_get_2d( - ffi.Pointer tensor, - ffi.Pointer data, - int offset, - int size, - int n_copies, - int stride_tensor, - int stride_data, + @Deprecated('use ggml_rope_ext_inplace instead') + ffi.Pointer ggml_rope_custom_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int n_dims, + int mode, + int n_ctx_orig, + double freq_base, + double freq_scale, + double ext_factor, + double attn_factor, + double beta_fast, + double beta_slow, ) { - return _ggml_backend_tensor_get_2d( - tensor, - data, - offset, - size, - n_copies, - stride_tensor, - stride_data, + return _ggml_rope_custom_inplace( + ctx, + a, + b, + n_dims, + mode, + n_ctx_orig, + freq_base, + freq_scale, + ext_factor, + attn_factor, + beta_fast, + beta_slow, ); } - late final _ggml_backend_tensor_get_2dPtr = _lookup< + late final _ggml_rope_custom_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function( + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size, - ffi.Size)>>('ggml_backend_tensor_get_2d'); - late final _ggml_backend_tensor_get_2d = - _ggml_backend_tensor_get_2dPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int, - int, int, int, int)>(); + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_rope_custom_inplace'); + late final _ggml_rope_custom_inplace = + _ggml_rope_custom_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + double, + double, + double, + double, + double, + double)>(); - void ggml_backend_tensor_memset( - ffi.Pointer tensor, - int value, - int offset, - int size, + ffi.Pointer ggml_rope_ext( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + int n_dims, + int mode, + int n_ctx_orig, + double freq_base, + double freq_scale, + double ext_factor, + double attn_factor, + double beta_fast, + double beta_slow, ) { - return _ggml_backend_tensor_memset( - tensor, - value, - offset, - size, + return _ggml_rope_ext( + ctx, + a, + b, + c, + n_dims, + mode, + n_ctx_orig, + freq_base, + freq_scale, + ext_factor, + attn_factor, + beta_fast, + beta_slow, ); } - late final _ggml_backend_tensor_memsetPtr = _lookup< + late final _ggml_rope_extPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Uint8, ffi.Size, - ffi.Size)>>('ggml_backend_tensor_memset'); - late final _ggml_backend_tensor_memset = _ggml_backend_tensor_memsetPtr - .asFunction, int, int, int)>(); - - void ggml_backend_synchronize( - ggml_backend_t backend, - ) { - return _ggml_backend_synchronize( - backend, - ); - } - - late final _ggml_backend_synchronizePtr = - _lookup>( - 'ggml_backend_synchronize'); - late final _ggml_backend_synchronize = - _ggml_backend_synchronizePtr.asFunction(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_rope_ext'); + late final _ggml_rope_ext = _ggml_rope_extPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + double, + double, + double, + double, + double, + double)>(); - ggml_backend_graph_plan_t ggml_backend_graph_plan_create( - ggml_backend_t backend, - ffi.Pointer cgraph, + ffi.Pointer ggml_rope_ext_back( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + int n_dims, + int mode, + int n_ctx_orig, + double freq_base, + double freq_scale, + double ext_factor, + double attn_factor, + double beta_fast, + double beta_slow, ) { - return _ggml_backend_graph_plan_create( - backend, - cgraph, + return _ggml_rope_ext_back( + ctx, + a, + b, + c, + n_dims, + mode, + n_ctx_orig, + freq_base, + freq_scale, + ext_factor, + attn_factor, + beta_fast, + beta_slow, ); } - late final _ggml_backend_graph_plan_createPtr = _lookup< + late final _ggml_rope_ext_backPtr = _lookup< ffi.NativeFunction< - ggml_backend_graph_plan_t Function(ggml_backend_t, - ffi.Pointer)>>('ggml_backend_graph_plan_create'); - late final _ggml_backend_graph_plan_create = - _ggml_backend_graph_plan_createPtr.asFunction< - ggml_backend_graph_plan_t Function( - ggml_backend_t, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_rope_ext_back'); + late final _ggml_rope_ext_back = _ggml_rope_ext_backPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + double, + double, + double, + double, + double, + double)>(); - void ggml_backend_graph_plan_free( - ggml_backend_t backend, - ggml_backend_graph_plan_t plan, + ffi.Pointer ggml_rope_ext_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + int n_dims, + int mode, + int n_ctx_orig, + double freq_base, + double freq_scale, + double ext_factor, + double attn_factor, + double beta_fast, + double beta_slow, ) { - return _ggml_backend_graph_plan_free( - backend, - plan, + return _ggml_rope_ext_inplace( + ctx, + a, + b, + c, + n_dims, + mode, + n_ctx_orig, + freq_base, + freq_scale, + ext_factor, + attn_factor, + beta_fast, + beta_slow, ); } - late final _ggml_backend_graph_plan_freePtr = _lookup< + late final _ggml_rope_ext_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_t, - ggml_backend_graph_plan_t)>>('ggml_backend_graph_plan_free'); - late final _ggml_backend_graph_plan_free = _ggml_backend_graph_plan_freePtr - .asFunction(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_rope_ext_inplace'); + late final _ggml_rope_ext_inplace = _ggml_rope_ext_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + double, + double, + double, + double, + double, + double)>(); - ggml_status ggml_backend_graph_plan_compute( - ggml_backend_t backend, - ggml_backend_graph_plan_t plan, + ffi.Pointer ggml_rope_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int n_dims, + int mode, ) { - return ggml_status.fromValue(_ggml_backend_graph_plan_compute( - backend, - plan, - )); + return _ggml_rope_inplace( + ctx, + a, + b, + n_dims, + mode, + ); } - late final _ggml_backend_graph_plan_computePtr = _lookup< + late final _ggml_rope_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ggml_backend_t, - ggml_backend_graph_plan_t)>>('ggml_backend_graph_plan_compute'); - late final _ggml_backend_graph_plan_compute = - _ggml_backend_graph_plan_computePtr.asFunction< - int Function(ggml_backend_t, ggml_backend_graph_plan_t)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int)>>('ggml_rope_inplace'); + late final _ggml_rope_inplace = _ggml_rope_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int, int)>(); - ggml_status ggml_backend_graph_compute( - ggml_backend_t backend, - ffi.Pointer cgraph, + ffi.Pointer ggml_rope_multi( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + int n_dims, + ffi.Pointer sections, + int mode, + int n_ctx_orig, + double freq_base, + double freq_scale, + double ext_factor, + double attn_factor, + double beta_fast, + double beta_slow, ) { - return ggml_status.fromValue(_ggml_backend_graph_compute( - backend, - cgraph, - )); + return _ggml_rope_multi( + ctx, + a, + b, + c, + n_dims, + sections, + mode, + n_ctx_orig, + freq_base, + freq_scale, + ext_factor, + attn_factor, + beta_fast, + beta_slow, + ); } - late final _ggml_backend_graph_computePtr = _lookup< + late final _ggml_rope_multiPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ggml_backend_t, - ffi.Pointer)>>('ggml_backend_graph_compute'); - late final _ggml_backend_graph_compute = _ggml_backend_graph_computePtr - .asFunction)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_rope_multi'); + late final _ggml_rope_multi = _ggml_rope_multiPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + ffi.Pointer, + int, + int, + double, + double, + double, + double, + double, + double)>(); - ggml_status ggml_backend_graph_compute_async( - ggml_backend_t backend, - ffi.Pointer cgraph, + ffi.Pointer ggml_rope_multi_back( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + int n_dims, + ffi.Pointer sections, + int mode, + int n_ctx_orig, + double freq_base, + double freq_scale, + double ext_factor, + double attn_factor, + double beta_fast, + double beta_slow, ) { - return ggml_status.fromValue(_ggml_backend_graph_compute_async( - backend, - cgraph, - )); + return _ggml_rope_multi_back( + ctx, + a, + b, + c, + n_dims, + sections, + mode, + n_ctx_orig, + freq_base, + freq_scale, + ext_factor, + attn_factor, + beta_fast, + beta_slow, + ); } - late final _ggml_backend_graph_compute_asyncPtr = _lookup< + late final _ggml_rope_multi_backPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ggml_backend_t, - ffi.Pointer)>>('ggml_backend_graph_compute_async'); - late final _ggml_backend_graph_compute_async = - _ggml_backend_graph_compute_asyncPtr - .asFunction)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_rope_multi_back'); + late final _ggml_rope_multi_back = _ggml_rope_multi_backPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + ffi.Pointer, + int, + int, + double, + double, + double, + double, + double, + double)>(); - bool ggml_backend_supports_op( - ggml_backend_t backend, - ffi.Pointer op, + ffi.Pointer ggml_rope_multi_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + int n_dims, + ffi.Pointer sections, + int mode, + int n_ctx_orig, + double freq_base, + double freq_scale, + double ext_factor, + double attn_factor, + double beta_fast, + double beta_slow, ) { - return _ggml_backend_supports_op( - backend, - op, + return _ggml_rope_multi_inplace( + ctx, + a, + b, + c, + n_dims, + sections, + mode, + n_ctx_orig, + freq_base, + freq_scale, + ext_factor, + attn_factor, + beta_fast, + beta_slow, ); } - late final _ggml_backend_supports_opPtr = _lookup< + late final _ggml_rope_multi_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_backend_t, - ffi.Pointer)>>('ggml_backend_supports_op'); - late final _ggml_backend_supports_op = _ggml_backend_supports_opPtr - .asFunction)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_rope_multi_inplace'); + late final _ggml_rope_multi_inplace = _ggml_rope_multi_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + ffi.Pointer, + int, + int, + double, + double, + double, + double, + double, + double)>(); - bool ggml_backend_supports_buft( - ggml_backend_t backend, - ggml_backend_buffer_type_t buft, + void ggml_rope_yarn_corr_dims( + int n_dims, + int n_ctx_orig, + double freq_base, + double beta_fast, + double beta_slow, + ffi.Pointer dims, ) { - return _ggml_backend_supports_buft( - backend, - buft, + return _ggml_rope_yarn_corr_dims( + n_dims, + n_ctx_orig, + freq_base, + beta_fast, + beta_slow, + dims, ); } - late final _ggml_backend_supports_buftPtr = _lookup< + late final _ggml_rope_yarn_corr_dimsPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_backend_t, - ggml_backend_buffer_type_t)>>('ggml_backend_supports_buft'); - late final _ggml_backend_supports_buft = _ggml_backend_supports_buftPtr - .asFunction(); + ffi.Void Function(ffi.Int, ffi.Int, ffi.Float, ffi.Float, ffi.Float, + ffi.Pointer)>>('ggml_rope_yarn_corr_dims'); + late final _ggml_rope_yarn_corr_dims = + _ggml_rope_yarn_corr_dimsPtr.asFunction< + void Function( + int, int, double, double, double, ffi.Pointer)>(); - bool ggml_backend_offload_op( - ggml_backend_t backend, - ffi.Pointer op, + ffi.Pointer ggml_round( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_offload_op( - backend, - op, + return _ggml_round( + ctx, + a, ); } - late final _ggml_backend_offload_opPtr = _lookup< + late final _ggml_roundPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_backend_t, - ffi.Pointer)>>('ggml_backend_offload_op'); - late final _ggml_backend_offload_op = _ggml_backend_offload_opPtr - .asFunction)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_round'); + late final _ggml_round = _ggml_roundPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_tensor_copy_async( - ggml_backend_t backend_src, - ggml_backend_t backend_dst, - ffi.Pointer src, - ffi.Pointer dst, + ffi.Pointer ggml_round_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_tensor_copy_async( - backend_src, - backend_dst, - src, - dst, - ); - } + return _ggml_round_inplace( + ctx, + a, + ); + } - late final _ggml_backend_tensor_copy_asyncPtr = _lookup< + late final _ggml_round_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_backend_t, - ggml_backend_t, - ffi.Pointer, - ffi.Pointer)>>('ggml_backend_tensor_copy_async'); - late final _ggml_backend_tensor_copy_async = - _ggml_backend_tensor_copy_asyncPtr.asFunction< - void Function(ggml_backend_t, ggml_backend_t, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_round_inplace'); + late final _ggml_round_inplace = _ggml_round_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_dev_t ggml_backend_get_device( - ggml_backend_t backend, + int ggml_row_size( + ggml_type type, + int ne, ) { - return _ggml_backend_get_device( - backend, + return _ggml_row_size( + type.value, + ne, ); } - late final _ggml_backend_get_devicePtr = - _lookup>( - 'ggml_backend_get_device'); - late final _ggml_backend_get_device = _ggml_backend_get_devicePtr - .asFunction(); + late final _ggml_row_sizePtr = _lookup< + ffi.NativeFunction>( + 'ggml_row_size'); + late final _ggml_row_size = + _ggml_row_sizePtr.asFunction(); - ggml_backend_event_t ggml_backend_event_new( - ggml_backend_dev_t device, + ffi.Pointer ggml_rwkv_wkv6( + ffi.Pointer ctx, + ffi.Pointer k, + ffi.Pointer v, + ffi.Pointer r, + ffi.Pointer tf, + ffi.Pointer td, + ffi.Pointer state, ) { - return _ggml_backend_event_new( - device, + return _ggml_rwkv_wkv6( + ctx, + k, + v, + r, + tf, + td, + state, ); } - late final _ggml_backend_event_newPtr = _lookup< - ffi - .NativeFunction>( - 'ggml_backend_event_new'); - late final _ggml_backend_event_new = _ggml_backend_event_newPtr - .asFunction(); + late final _ggml_rwkv_wkv6Ptr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_rwkv_wkv6'); + late final _ggml_rwkv_wkv6 = _ggml_rwkv_wkv6Ptr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - void ggml_backend_event_free( - ggml_backend_event_t event, + ffi.Pointer ggml_rwkv_wkv7( + ffi.Pointer ctx, + ffi.Pointer r, + ffi.Pointer w, + ffi.Pointer k, + ffi.Pointer v, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer state, ) { - return _ggml_backend_event_free( - event, + return _ggml_rwkv_wkv7( + ctx, + r, + w, + k, + v, + a, + b, + state, ); } - late final _ggml_backend_event_freePtr = - _lookup>( - 'ggml_backend_event_free'); - late final _ggml_backend_event_free = _ggml_backend_event_freePtr - .asFunction(); + late final _ggml_rwkv_wkv7Ptr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_rwkv_wkv7'); + late final _ggml_rwkv_wkv7 = _ggml_rwkv_wkv7Ptr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - void ggml_backend_event_record( - ggml_backend_event_t event, - ggml_backend_t backend, + ffi.Pointer ggml_scale( + ffi.Pointer ctx, + ffi.Pointer a, + double s, ) { - return _ggml_backend_event_record( - event, - backend, + return _ggml_scale( + ctx, + a, + s, ); } - late final _ggml_backend_event_recordPtr = _lookup< + late final _ggml_scalePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_event_t, - ggml_backend_t)>>('ggml_backend_event_record'); - late final _ggml_backend_event_record = _ggml_backend_event_recordPtr - .asFunction(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_scale'); + late final _ggml_scale = _ggml_scalePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, double)>(); - void ggml_backend_event_synchronize( - ggml_backend_event_t event, + ffi.Pointer ggml_scale_bias( + ffi.Pointer ctx, + ffi.Pointer a, + double s, + double b, ) { - return _ggml_backend_event_synchronize( - event, + return _ggml_scale_bias( + ctx, + a, + s, + b, ); } - late final _ggml_backend_event_synchronizePtr = - _lookup>( - 'ggml_backend_event_synchronize'); - late final _ggml_backend_event_synchronize = - _ggml_backend_event_synchronizePtr - .asFunction(); + late final _ggml_scale_biasPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float)>>('ggml_scale_bias'); + late final _ggml_scale_bias = _ggml_scale_biasPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, double, double)>(); - void ggml_backend_event_wait( - ggml_backend_t backend, - ggml_backend_event_t event, + ffi.Pointer ggml_scale_bias_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + double s, + double b, ) { - return _ggml_backend_event_wait( - backend, - event, + return _ggml_scale_bias_inplace( + ctx, + a, + s, + b, ); } - late final _ggml_backend_event_waitPtr = _lookup< + late final _ggml_scale_bias_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_t, - ggml_backend_event_t)>>('ggml_backend_event_wait'); - late final _ggml_backend_event_wait = _ggml_backend_event_waitPtr - .asFunction(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float)>>('ggml_scale_bias_inplace'); + late final _ggml_scale_bias_inplace = _ggml_scale_bias_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, double, double)>(); - ffi.Pointer ggml_backend_dev_name( - ggml_backend_dev_t device, + ffi.Pointer ggml_scale_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + double s, ) { - return _ggml_backend_dev_name( - device, + return _ggml_scale_inplace( + ctx, + a, + s, ); } - late final _ggml_backend_dev_namePtr = _lookup< - ffi - .NativeFunction Function(ggml_backend_dev_t)>>( - 'ggml_backend_dev_name'); - late final _ggml_backend_dev_name = _ggml_backend_dev_namePtr - .asFunction Function(ggml_backend_dev_t)>(); + late final _ggml_scale_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Float)>>('ggml_scale_inplace'); + late final _ggml_scale_inplace = _ggml_scale_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, double)>(); - ffi.Pointer ggml_backend_dev_description( - ggml_backend_dev_t device, + ffi.Pointer ggml_set( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int nb1, + int nb2, + int nb3, + int offset, ) { - return _ggml_backend_dev_description( - device, + return _ggml_set( + ctx, + a, + b, + nb1, + nb2, + nb3, + offset, ); } - late final _ggml_backend_dev_descriptionPtr = _lookup< - ffi - .NativeFunction Function(ggml_backend_dev_t)>>( - 'ggml_backend_dev_description'); - late final _ggml_backend_dev_description = _ggml_backend_dev_descriptionPtr - .asFunction Function(ggml_backend_dev_t)>(); + late final _ggml_setPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_set'); + late final _ggml_set = _ggml_setPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int)>(); - void ggml_backend_dev_memory( - ggml_backend_dev_t device, - ffi.Pointer free, - ffi.Pointer total, + ffi.Pointer ggml_set_1d( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int offset, ) { - return _ggml_backend_dev_memory( - device, - free, - total, + return _ggml_set_1d( + ctx, + a, + b, + offset, ); } - late final _ggml_backend_dev_memoryPtr = _lookup< + late final _ggml_set_1dPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_dev_t, ffi.Pointer, - ffi.Pointer)>>('ggml_backend_dev_memory'); - late final _ggml_backend_dev_memory = _ggml_backend_dev_memoryPtr.asFunction< - void Function( - ggml_backend_dev_t, ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size)>>('ggml_set_1d'); + late final _ggml_set_1d = _ggml_set_1dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int)>(); - ggml_backend_dev_type ggml_backend_dev_type$1( - ggml_backend_dev_t device, + ffi.Pointer ggml_set_1d_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int offset, ) { - return ggml_backend_dev_type.fromValue(_ggml_backend_dev_type$1( - device, - )); + return _ggml_set_1d_inplace( + ctx, + a, + b, + offset, + ); } - late final _ggml_backend_dev_type$1Ptr = - _lookup>( - 'ggml_backend_dev_type'); - late final _ggml_backend_dev_type$1 = _ggml_backend_dev_type$1Ptr - .asFunction(); + late final _ggml_set_1d_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size)>>('ggml_set_1d_inplace'); + late final _ggml_set_1d_inplace = _ggml_set_1d_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int)>(); - void ggml_backend_dev_get_props( - ggml_backend_dev_t device, - ffi.Pointer props, + ffi.Pointer ggml_set_2d( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int nb1, + int offset, ) { - return _ggml_backend_dev_get_props( - device, - props, + return _ggml_set_2d( + ctx, + a, + b, + nb1, + offset, ); } - late final _ggml_backend_dev_get_propsPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ggml_backend_dev_t, ffi.Pointer)>>( - 'ggml_backend_dev_get_props'); - late final _ggml_backend_dev_get_props = - _ggml_backend_dev_get_propsPtr.asFunction< - void Function( - ggml_backend_dev_t, ffi.Pointer)>(); + late final _ggml_set_2dPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size)>>('ggml_set_2d'); + late final _ggml_set_2d = _ggml_set_2dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int, int)>(); - ggml_backend_reg_t ggml_backend_dev_backend_reg( - ggml_backend_dev_t device, + ffi.Pointer ggml_set_2d_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int nb1, + int offset, ) { - return _ggml_backend_dev_backend_reg( - device, + return _ggml_set_2d_inplace( + ctx, + a, + b, + nb1, + offset, ); } - late final _ggml_backend_dev_backend_regPtr = _lookup< - ffi.NativeFunction>( - 'ggml_backend_dev_backend_reg'); - late final _ggml_backend_dev_backend_reg = _ggml_backend_dev_backend_regPtr - .asFunction(); + late final _ggml_set_2d_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size)>>('ggml_set_2d_inplace'); + late final _ggml_set_2d_inplace = _ggml_set_2d_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer, int, int)>(); - ggml_backend_t ggml_backend_dev_init( - ggml_backend_dev_t device, - ffi.Pointer params, + ggml_abort_callback_t ggml_set_abort_callback( + ggml_abort_callback_t callback, ) { - return _ggml_backend_dev_init( - device, - params, + return _ggml_set_abort_callback( + callback, ); } - late final _ggml_backend_dev_initPtr = _lookup< + late final _ggml_set_abort_callbackPtr = _lookup< ffi.NativeFunction< - ggml_backend_t Function(ggml_backend_dev_t, - ffi.Pointer)>>('ggml_backend_dev_init'); - late final _ggml_backend_dev_init = _ggml_backend_dev_initPtr.asFunction< - ggml_backend_t Function(ggml_backend_dev_t, ffi.Pointer)>(); + ggml_abort_callback_t Function( + ggml_abort_callback_t)>>('ggml_set_abort_callback'); + late final _ggml_set_abort_callback = _ggml_set_abort_callbackPtr + .asFunction(); - ggml_backend_buffer_type_t ggml_backend_dev_buffer_type( - ggml_backend_dev_t device, + ffi.Pointer ggml_set_f32( + ffi.Pointer tensor, + double value, ) { - return _ggml_backend_dev_buffer_type( - device, + return _ggml_set_f32( + tensor, + value, ); } - late final _ggml_backend_dev_buffer_typePtr = _lookup< + late final _ggml_set_f32Ptr = _lookup< ffi.NativeFunction< - ggml_backend_buffer_type_t Function( - ggml_backend_dev_t)>>('ggml_backend_dev_buffer_type'); - late final _ggml_backend_dev_buffer_type = _ggml_backend_dev_buffer_typePtr - .asFunction(); + ffi.Pointer Function( + ffi.Pointer, ffi.Float)>>('ggml_set_f32'); + late final _ggml_set_f32 = _ggml_set_f32Ptr.asFunction< + ffi.Pointer Function(ffi.Pointer, double)>(); - ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type( - ggml_backend_dev_t device, + void ggml_set_f32_1d( + ffi.Pointer tensor, + int i, + double value, ) { - return _ggml_backend_dev_host_buffer_type( - device, + return _ggml_set_f32_1d( + tensor, + i, + value, ); } - late final _ggml_backend_dev_host_buffer_typePtr = _lookup< + late final _ggml_set_f32_1dPtr = _lookup< ffi.NativeFunction< - ggml_backend_buffer_type_t Function( - ggml_backend_dev_t)>>('ggml_backend_dev_host_buffer_type'); - late final _ggml_backend_dev_host_buffer_type = - _ggml_backend_dev_host_buffer_typePtr.asFunction< - ggml_backend_buffer_type_t Function(ggml_backend_dev_t)>(); + ffi.Void Function(ffi.Pointer, ffi.Int, + ffi.Float)>>('ggml_set_f32_1d'); + late final _ggml_set_f32_1d = _ggml_set_f32_1dPtr + .asFunction, int, double)>(); - ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr( - ggml_backend_dev_t device, - ffi.Pointer ptr, - int size, - int max_tensor_size, + void ggml_set_f32_nd( + ffi.Pointer tensor, + int i0, + int i1, + int i2, + int i3, + double value, ) { - return _ggml_backend_dev_buffer_from_host_ptr( - device, - ptr, - size, - max_tensor_size, + return _ggml_set_f32_nd( + tensor, + i0, + i1, + i2, + i3, + value, ); } - late final _ggml_backend_dev_buffer_from_host_ptrPtr = _lookup< + late final _ggml_set_f32_ndPtr = _lookup< ffi.NativeFunction< - ggml_backend_buffer_t Function( - ggml_backend_dev_t, - ffi.Pointer, - ffi.Size, - ffi.Size)>>('ggml_backend_dev_buffer_from_host_ptr'); - late final _ggml_backend_dev_buffer_from_host_ptr = - _ggml_backend_dev_buffer_from_host_ptrPtr.asFunction< - ggml_backend_buffer_t Function( - ggml_backend_dev_t, ffi.Pointer, int, int)>(); + ffi.Void Function(ffi.Pointer, ffi.Int, ffi.Int, ffi.Int, + ffi.Int, ffi.Float)>>('ggml_set_f32_nd'); + late final _ggml_set_f32_nd = _ggml_set_f32_ndPtr.asFunction< + void Function(ffi.Pointer, int, int, int, int, double)>(); - bool ggml_backend_dev_supports_op( - ggml_backend_dev_t device, - ffi.Pointer op, + ffi.Pointer ggml_set_i32( + ffi.Pointer tensor, + int value, ) { - return _ggml_backend_dev_supports_op( - device, - op, + return _ggml_set_i32( + tensor, + value, ); } - late final _ggml_backend_dev_supports_opPtr = _lookup< + late final _ggml_set_i32Ptr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_backend_dev_t, - ffi.Pointer)>>('ggml_backend_dev_supports_op'); - late final _ggml_backend_dev_supports_op = - _ggml_backend_dev_supports_opPtr.asFunction< - bool Function(ggml_backend_dev_t, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, ffi.Int32)>>('ggml_set_i32'); + late final _ggml_set_i32 = _ggml_set_i32Ptr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - bool ggml_backend_dev_supports_buft( - ggml_backend_dev_t device, - ggml_backend_buffer_type_t buft, + void ggml_set_i32_1d( + ffi.Pointer tensor, + int i, + int value, ) { - return _ggml_backend_dev_supports_buft( - device, - buft, + return _ggml_set_i32_1d( + tensor, + i, + value, ); } - late final _ggml_backend_dev_supports_buftPtr = _lookup< + late final _ggml_set_i32_1dPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_backend_dev_t, - ggml_backend_buffer_type_t)>>('ggml_backend_dev_supports_buft'); - late final _ggml_backend_dev_supports_buft = - _ggml_backend_dev_supports_buftPtr.asFunction< - bool Function(ggml_backend_dev_t, ggml_backend_buffer_type_t)>(); + ffi.Void Function(ffi.Pointer, ffi.Int, + ffi.Int32)>>('ggml_set_i32_1d'); + late final _ggml_set_i32_1d = _ggml_set_i32_1dPtr + .asFunction, int, int)>(); - bool ggml_backend_dev_offload_op( - ggml_backend_dev_t device, - ffi.Pointer op, + void ggml_set_i32_nd( + ffi.Pointer tensor, + int i0, + int i1, + int i2, + int i3, + int value, ) { - return _ggml_backend_dev_offload_op( - device, - op, + return _ggml_set_i32_nd( + tensor, + i0, + i1, + i2, + i3, + value, ); } - late final _ggml_backend_dev_offload_opPtr = _lookup< + late final _ggml_set_i32_ndPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_backend_dev_t, - ffi.Pointer)>>('ggml_backend_dev_offload_op'); - late final _ggml_backend_dev_offload_op = - _ggml_backend_dev_offload_opPtr.asFunction< - bool Function(ggml_backend_dev_t, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Int, ffi.Int, ffi.Int, + ffi.Int, ffi.Int32)>>('ggml_set_i32_nd'); + late final _ggml_set_i32_nd = _ggml_set_i32_ndPtr.asFunction< + void Function(ffi.Pointer, int, int, int, int, int)>(); - ffi.Pointer ggml_backend_reg_name( - ggml_backend_reg_t reg, + ffi.Pointer ggml_set_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + int nb1, + int nb2, + int nb3, + int offset, ) { - return _ggml_backend_reg_name( - reg, + return _ggml_set_inplace( + ctx, + a, + b, + nb1, + nb2, + nb3, + offset, ); } - late final _ggml_backend_reg_namePtr = _lookup< - ffi - .NativeFunction Function(ggml_backend_reg_t)>>( - 'ggml_backend_reg_name'); - late final _ggml_backend_reg_name = _ggml_backend_reg_namePtr - .asFunction Function(ggml_backend_reg_t)>(); + late final _ggml_set_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_set_inplace'); + late final _ggml_set_inplace = _ggml_set_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + int)>(); - int ggml_backend_reg_dev_count( - ggml_backend_reg_t reg, + void ggml_set_input( + ffi.Pointer tensor, ) { - return _ggml_backend_reg_dev_count( - reg, + return _ggml_set_input( + tensor, ); } - late final _ggml_backend_reg_dev_countPtr = - _lookup>( - 'ggml_backend_reg_dev_count'); - late final _ggml_backend_reg_dev_count = _ggml_backend_reg_dev_countPtr - .asFunction(); + late final _ggml_set_inputPtr = + _lookup)>>( + 'ggml_set_input'); + late final _ggml_set_input = + _ggml_set_inputPtr.asFunction)>(); - ggml_backend_dev_t ggml_backend_reg_dev_get( - ggml_backend_reg_t reg, - int index, + void ggml_set_loss( + ffi.Pointer tensor, ) { - return _ggml_backend_reg_dev_get( - reg, - index, + return _ggml_set_loss( + tensor, ); } - late final _ggml_backend_reg_dev_getPtr = _lookup< - ffi.NativeFunction< - ggml_backend_dev_t Function( - ggml_backend_reg_t, ffi.Size)>>('ggml_backend_reg_dev_get'); - late final _ggml_backend_reg_dev_get = _ggml_backend_reg_dev_getPtr - .asFunction(); + late final _ggml_set_lossPtr = + _lookup)>>( + 'ggml_set_loss'); + late final _ggml_set_loss = + _ggml_set_lossPtr.asFunction)>(); - ffi.Pointer ggml_backend_reg_get_proc_address( - ggml_backend_reg_t reg, + ffi.Pointer ggml_set_name( + ffi.Pointer tensor, ffi.Pointer name, ) { - return _ggml_backend_reg_get_proc_address( - reg, + return _ggml_set_name( + tensor, name, ); } - late final _ggml_backend_reg_get_proc_addressPtr = _lookup< + late final _ggml_set_namePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ggml_backend_reg_t, - ffi.Pointer)>>('ggml_backend_reg_get_proc_address'); - late final _ggml_backend_reg_get_proc_address = - _ggml_backend_reg_get_proc_addressPtr.asFunction< - ffi.Pointer Function( - ggml_backend_reg_t, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_set_name'); + late final _ggml_set_name = _ggml_set_namePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_register( - ggml_backend_reg_t reg, + void ggml_set_no_alloc( + ffi.Pointer ctx, + bool no_alloc, ) { - return _ggml_backend_register( - reg, + return _ggml_set_no_alloc( + ctx, + no_alloc, ); } - late final _ggml_backend_registerPtr = - _lookup>( - 'ggml_backend_register'); - late final _ggml_backend_register = - _ggml_backend_registerPtr.asFunction(); + late final _ggml_set_no_allocPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, ffi.Bool)>>('ggml_set_no_alloc'); + late final _ggml_set_no_alloc = _ggml_set_no_allocPtr + .asFunction, bool)>(); - void ggml_backend_device_register( - ggml_backend_dev_t device, + void ggml_set_output( + ffi.Pointer tensor, ) { - return _ggml_backend_device_register( - device, + return _ggml_set_output( + tensor, ); } - late final _ggml_backend_device_registerPtr = - _lookup>( - 'ggml_backend_device_register'); - late final _ggml_backend_device_register = _ggml_backend_device_registerPtr - .asFunction(); - - int ggml_backend_reg_count() { - return _ggml_backend_reg_count(); - } - - late final _ggml_backend_reg_countPtr = - _lookup>( - 'ggml_backend_reg_count'); - late final _ggml_backend_reg_count = - _ggml_backend_reg_countPtr.asFunction(); + late final _ggml_set_outputPtr = + _lookup)>>( + 'ggml_set_output'); + late final _ggml_set_output = + _ggml_set_outputPtr.asFunction)>(); - ggml_backend_reg_t ggml_backend_reg_get( - int index, + void ggml_set_param( + ffi.Pointer tensor, ) { - return _ggml_backend_reg_get( - index, + return _ggml_set_param( + tensor, ); } - late final _ggml_backend_reg_getPtr = - _lookup>( - 'ggml_backend_reg_get'); - late final _ggml_backend_reg_get = - _ggml_backend_reg_getPtr.asFunction(); + late final _ggml_set_paramPtr = + _lookup)>>( + 'ggml_set_param'); + late final _ggml_set_param = + _ggml_set_paramPtr.asFunction)>(); - ggml_backend_reg_t ggml_backend_reg_by_name( - ffi.Pointer name, + ffi.Pointer ggml_set_rows( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, ) { - return _ggml_backend_reg_by_name( - name, + return _ggml_set_rows( + ctx, + a, + b, + c, ); } - late final _ggml_backend_reg_by_namePtr = _lookup< - ffi - .NativeFunction)>>( - 'ggml_backend_reg_by_name'); - late final _ggml_backend_reg_by_name = _ggml_backend_reg_by_namePtr - .asFunction)>(); + late final _ggml_set_rowsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_set_rows'); + late final _ggml_set_rows = _ggml_set_rowsPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - int ggml_backend_dev_count() { - return _ggml_backend_dev_count(); + ffi.Pointer ggml_set_zero( + ffi.Pointer tensor, + ) { + return _ggml_set_zero( + tensor, + ); } - late final _ggml_backend_dev_countPtr = - _lookup>( - 'ggml_backend_dev_count'); - late final _ggml_backend_dev_count = - _ggml_backend_dev_countPtr.asFunction(); + late final _ggml_set_zeroPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer)>>('ggml_set_zero'); + late final _ggml_set_zero = _ggml_set_zeroPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - ggml_backend_dev_t ggml_backend_dev_get( - int index, + ffi.Pointer ggml_sgn( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_dev_get( - index, + return _ggml_sgn( + ctx, + a, ); } - late final _ggml_backend_dev_getPtr = - _lookup>( - 'ggml_backend_dev_get'); - late final _ggml_backend_dev_get = - _ggml_backend_dev_getPtr.asFunction(); + late final _ggml_sgnPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sgn'); + late final _ggml_sgn = _ggml_sgnPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_dev_t ggml_backend_dev_by_name( - ffi.Pointer name, + ffi.Pointer ggml_sgn_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_dev_by_name( - name, + return _ggml_sgn_inplace( + ctx, + a, ); } - late final _ggml_backend_dev_by_namePtr = _lookup< - ffi - .NativeFunction)>>( - 'ggml_backend_dev_by_name'); - late final _ggml_backend_dev_by_name = _ggml_backend_dev_by_namePtr - .asFunction)>(); + late final _ggml_sgn_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sgn_inplace'); + late final _ggml_sgn_inplace = _ggml_sgn_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_dev_t ggml_backend_dev_by_type( - ggml_backend_dev_type type, + ffi.Pointer ggml_sigmoid( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_dev_by_type( - type.value, + return _ggml_sigmoid( + ctx, + a, ); } - late final _ggml_backend_dev_by_typePtr = - _lookup>( - 'ggml_backend_dev_by_type'); - late final _ggml_backend_dev_by_type = _ggml_backend_dev_by_typePtr - .asFunction(); + late final _ggml_sigmoidPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sigmoid'); + late final _ggml_sigmoid = _ggml_sigmoidPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_t ggml_backend_init_by_name( - ffi.Pointer name, - ffi.Pointer params, + ffi.Pointer ggml_sigmoid_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_init_by_name( - name, - params, + return _ggml_sigmoid_inplace( + ctx, + a, ); } - late final _ggml_backend_init_by_namePtr = _lookup< + late final _ggml_sigmoid_inplacePtr = _lookup< ffi.NativeFunction< - ggml_backend_t Function(ffi.Pointer, - ffi.Pointer)>>('ggml_backend_init_by_name'); - late final _ggml_backend_init_by_name = - _ggml_backend_init_by_namePtr.asFunction< - ggml_backend_t Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sigmoid_inplace'); + late final _ggml_sigmoid_inplace = _ggml_sigmoid_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_t ggml_backend_init_by_type( - ggml_backend_dev_type type, - ffi.Pointer params, + ffi.Pointer ggml_silu( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_init_by_type( - type.value, - params, + return _ggml_silu( + ctx, + a, ); } - late final _ggml_backend_init_by_typePtr = _lookup< + late final _ggml_siluPtr = _lookup< ffi.NativeFunction< - ggml_backend_t Function(ffi.UnsignedInt, - ffi.Pointer)>>('ggml_backend_init_by_type'); - late final _ggml_backend_init_by_type = _ggml_backend_init_by_typePtr - .asFunction)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_silu'); + late final _ggml_silu = _ggml_siluPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_t ggml_backend_init_best() { - return _ggml_backend_init_best(); + ffi.Pointer ggml_silu_back( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + ) { + return _ggml_silu_back( + ctx, + a, + b, + ); } - late final _ggml_backend_init_bestPtr = - _lookup>( - 'ggml_backend_init_best'); - late final _ggml_backend_init_best = - _ggml_backend_init_bestPtr.asFunction(); + late final _ggml_silu_backPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_silu_back'); + late final _ggml_silu_back = _ggml_silu_backPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_reg_t ggml_backend_load( - ffi.Pointer path, + ffi.Pointer ggml_silu_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_load( - path, + return _ggml_silu_inplace( + ctx, + a, ); } - late final _ggml_backend_loadPtr = _lookup< - ffi - .NativeFunction)>>( - 'ggml_backend_load'); - late final _ggml_backend_load = _ggml_backend_loadPtr - .asFunction)>(); + late final _ggml_silu_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_silu_inplace'); + late final _ggml_silu_inplace = _ggml_silu_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_unload( - ggml_backend_reg_t reg, + ffi.Pointer ggml_sin( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_unload( - reg, + return _ggml_sin( + ctx, + a, ); } - late final _ggml_backend_unloadPtr = - _lookup>( - 'ggml_backend_unload'); - late final _ggml_backend_unload = - _ggml_backend_unloadPtr.asFunction(); + late final _ggml_sinPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sin'); + late final _ggml_sin = _ggml_sinPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_load_all() { - return _ggml_backend_load_all(); + ffi.Pointer ggml_sin_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ) { + return _ggml_sin_inplace( + ctx, + a, + ); } - late final _ggml_backend_load_allPtr = - _lookup>('ggml_backend_load_all'); - late final _ggml_backend_load_all = - _ggml_backend_load_allPtr.asFunction(); + late final _ggml_sin_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sin_inplace'); + late final _ggml_sin_inplace = _ggml_sin_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_load_all_from_path( - ffi.Pointer dir_path, + ffi.Pointer ggml_soft_max( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_load_all_from_path( - dir_path, + return _ggml_soft_max( + ctx, + a, ); } - late final _ggml_backend_load_all_from_pathPtr = - _lookup)>>( - 'ggml_backend_load_all_from_path'); - late final _ggml_backend_load_all_from_path = - _ggml_backend_load_all_from_pathPtr - .asFunction)>(); + late final _ggml_soft_maxPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_soft_max'); + late final _ggml_soft_max = _ggml_soft_maxPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_sched_t ggml_backend_sched_new( - ffi.Pointer backends, - ffi.Pointer bufts, - int n_backends, - int graph_size, - bool parallel, - bool op_offload, + void ggml_soft_max_add_sinks( + ffi.Pointer a, + ffi.Pointer sinks, ) { - return _ggml_backend_sched_new( - backends, - bufts, - n_backends, - graph_size, - parallel, - op_offload, + return _ggml_soft_max_add_sinks( + a, + sinks, ); } - late final _ggml_backend_sched_newPtr = _lookup< + late final _ggml_soft_max_add_sinksPtr = _lookup< ffi.NativeFunction< - ggml_backend_sched_t Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int, - ffi.Size, - ffi.Bool, - ffi.Bool)>>('ggml_backend_sched_new'); - late final _ggml_backend_sched_new = _ggml_backend_sched_newPtr.asFunction< - ggml_backend_sched_t Function(ffi.Pointer, - ffi.Pointer, int, int, bool, bool)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('ggml_soft_max_add_sinks'); + late final _ggml_soft_max_add_sinks = _ggml_soft_max_add_sinksPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_sched_free( - ggml_backend_sched_t sched, + ffi.Pointer ggml_soft_max_ext( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer mask, + double scale, + double max_bias, ) { - return _ggml_backend_sched_free( - sched, + return _ggml_soft_max_ext( + ctx, + a, + mask, + scale, + max_bias, ); } - late final _ggml_backend_sched_freePtr = - _lookup>( - 'ggml_backend_sched_free'); - late final _ggml_backend_sched_free = _ggml_backend_sched_freePtr - .asFunction(); + late final _ggml_soft_max_extPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float)>>('ggml_soft_max_ext'); + late final _ggml_soft_max_ext = _ggml_soft_max_extPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + double, + double)>(); - void ggml_backend_sched_reserve_size( - ggml_backend_sched_t sched, - ffi.Pointer measure_graph, - ffi.Pointer sizes, + ffi.Pointer ggml_soft_max_ext_back( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + double scale, + double max_bias, ) { - return _ggml_backend_sched_reserve_size( - sched, - measure_graph, - sizes, + return _ggml_soft_max_ext_back( + ctx, + a, + b, + scale, + max_bias, ); } - late final _ggml_backend_sched_reserve_sizePtr = _lookup< + late final _ggml_soft_max_ext_backPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_sched_t, ffi.Pointer, - ffi.Pointer)>>('ggml_backend_sched_reserve_size'); - late final _ggml_backend_sched_reserve_size = - _ggml_backend_sched_reserve_sizePtr.asFunction< - void Function(ggml_backend_sched_t, ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float)>>('ggml_soft_max_ext_back'); + late final _ggml_soft_max_ext_back = _ggml_soft_max_ext_backPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + double, + double)>(); - bool ggml_backend_sched_reserve( - ggml_backend_sched_t sched, - ffi.Pointer measure_graph, + ffi.Pointer ggml_soft_max_ext_back_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + double scale, + double max_bias, ) { - return _ggml_backend_sched_reserve( - sched, - measure_graph, + return _ggml_soft_max_ext_back_inplace( + ctx, + a, + b, + scale, + max_bias, ); } - late final _ggml_backend_sched_reservePtr = _lookup< + late final _ggml_soft_max_ext_back_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_backend_sched_t, - ffi.Pointer)>>('ggml_backend_sched_reserve'); - late final _ggml_backend_sched_reserve = - _ggml_backend_sched_reservePtr.asFunction< - bool Function(ggml_backend_sched_t, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float)>>('ggml_soft_max_ext_back_inplace'); + late final _ggml_soft_max_ext_back_inplace = + _ggml_soft_max_ext_back_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + double, + double)>(); - int ggml_backend_sched_get_n_backends( - ggml_backend_sched_t sched, + ffi.Pointer ggml_soft_max_ext_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer mask, + double scale, + double max_bias, ) { - return _ggml_backend_sched_get_n_backends( - sched, + return _ggml_soft_max_ext_inplace( + ctx, + a, + mask, + scale, + max_bias, ); } - late final _ggml_backend_sched_get_n_backendsPtr = - _lookup>( - 'ggml_backend_sched_get_n_backends'); - late final _ggml_backend_sched_get_n_backends = - _ggml_backend_sched_get_n_backendsPtr - .asFunction(); + late final _ggml_soft_max_ext_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float)>>('ggml_soft_max_ext_inplace'); + late final _ggml_soft_max_ext_inplace = + _ggml_soft_max_ext_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + double, + double)>(); - ggml_backend_t ggml_backend_sched_get_backend( - ggml_backend_sched_t sched, - int i, + ffi.Pointer ggml_soft_max_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_get_backend( - sched, - i, + return _ggml_soft_max_inplace( + ctx, + a, ); } - late final _ggml_backend_sched_get_backendPtr = _lookup< + late final _ggml_soft_max_inplacePtr = _lookup< ffi.NativeFunction< - ggml_backend_t Function(ggml_backend_sched_t, - ffi.Int)>>('ggml_backend_sched_get_backend'); - late final _ggml_backend_sched_get_backend = - _ggml_backend_sched_get_backendPtr - .asFunction(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_soft_max_inplace'); + late final _ggml_soft_max_inplace = _ggml_soft_max_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - int ggml_backend_sched_get_n_splits( - ggml_backend_sched_t sched, + ffi.Pointer ggml_softplus( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_get_n_splits( - sched, + return _ggml_softplus( + ctx, + a, ); } - late final _ggml_backend_sched_get_n_splitsPtr = - _lookup>( - 'ggml_backend_sched_get_n_splits'); - late final _ggml_backend_sched_get_n_splits = - _ggml_backend_sched_get_n_splitsPtr - .asFunction(); + late final _ggml_softplusPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_softplus'); + late final _ggml_softplus = _ggml_softplusPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - int ggml_backend_sched_get_n_copies( - ggml_backend_sched_t sched, + ffi.Pointer ggml_softplus_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_get_n_copies( - sched, + return _ggml_softplus_inplace( + ctx, + a, ); } - late final _ggml_backend_sched_get_n_copiesPtr = - _lookup>( - 'ggml_backend_sched_get_n_copies'); - late final _ggml_backend_sched_get_n_copies = - _ggml_backend_sched_get_n_copiesPtr - .asFunction(); + late final _ggml_softplus_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_softplus_inplace'); + late final _ggml_softplus_inplace = _ggml_softplus_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type( - ggml_backend_sched_t sched, - ggml_backend_t backend, + ffi.Pointer ggml_solve_tri( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + bool left, + bool lower, + bool uni, ) { - return _ggml_backend_sched_get_buffer_type( - sched, - backend, + return _ggml_solve_tri( + ctx, + a, + b, + left, + lower, + uni, ); } - late final _ggml_backend_sched_get_buffer_typePtr = _lookup< + late final _ggml_solve_triPtr = _lookup< ffi.NativeFunction< - ggml_backend_buffer_type_t Function(ggml_backend_sched_t, - ggml_backend_t)>>('ggml_backend_sched_get_buffer_type'); - late final _ggml_backend_sched_get_buffer_type = - _ggml_backend_sched_get_buffer_typePtr.asFunction< - ggml_backend_buffer_type_t Function( - ggml_backend_sched_t, ggml_backend_t)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Bool, + ffi.Bool, + ffi.Bool)>>('ggml_solve_tri'); + late final _ggml_solve_tri = _ggml_solve_triPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + bool, + bool, + bool)>(); - int ggml_backend_sched_get_buffer_size( - ggml_backend_sched_t sched, - ggml_backend_t backend, + ffi.Pointer ggml_sqr( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_get_buffer_size( - sched, - backend, + return _ggml_sqr( + ctx, + a, ); } - late final _ggml_backend_sched_get_buffer_sizePtr = _lookup< + late final _ggml_sqrPtr = _lookup< ffi.NativeFunction< - ffi.Size Function(ggml_backend_sched_t, - ggml_backend_t)>>('ggml_backend_sched_get_buffer_size'); - late final _ggml_backend_sched_get_buffer_size = - _ggml_backend_sched_get_buffer_sizePtr - .asFunction(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sqr'); + late final _ggml_sqr = _ggml_sqrPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_sched_set_tensor_backend( - ggml_backend_sched_t sched, - ffi.Pointer node, - ggml_backend_t backend, + ffi.Pointer ggml_sqr_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_set_tensor_backend( - sched, - node, - backend, + return _ggml_sqr_inplace( + ctx, + a, ); } - late final _ggml_backend_sched_set_tensor_backendPtr = _lookup< + late final _ggml_sqr_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_sched_t, ffi.Pointer, - ggml_backend_t)>>('ggml_backend_sched_set_tensor_backend'); - late final _ggml_backend_sched_set_tensor_backend = - _ggml_backend_sched_set_tensor_backendPtr.asFunction< - void Function(ggml_backend_sched_t, ffi.Pointer, - ggml_backend_t)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sqr_inplace'); + late final _ggml_sqr_inplace = _ggml_sqr_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_t ggml_backend_sched_get_tensor_backend( - ggml_backend_sched_t sched, - ffi.Pointer node, + ffi.Pointer ggml_sqrt( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_get_tensor_backend( - sched, - node, + return _ggml_sqrt( + ctx, + a, ); } - late final _ggml_backend_sched_get_tensor_backendPtr = _lookup< - ffi.NativeFunction< - ggml_backend_t Function( - ggml_backend_sched_t, ffi.Pointer)>>( - 'ggml_backend_sched_get_tensor_backend'); - late final _ggml_backend_sched_get_tensor_backend = - _ggml_backend_sched_get_tensor_backendPtr.asFunction< - ggml_backend_t Function( - ggml_backend_sched_t, ffi.Pointer)>(); + late final _ggml_sqrtPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sqrt'); + late final _ggml_sqrt = _ggml_sqrtPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_sched_split_graph( - ggml_backend_sched_t sched, - ffi.Pointer graph, + ffi.Pointer ggml_sqrt_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_split_graph( - sched, - graph, + return _ggml_sqrt_inplace( + ctx, + a, ); } - late final _ggml_backend_sched_split_graphPtr = _lookup< + late final _ggml_sqrt_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_sched_t, - ffi.Pointer)>>('ggml_backend_sched_split_graph'); - late final _ggml_backend_sched_split_graph = - _ggml_backend_sched_split_graphPtr.asFunction< - void Function(ggml_backend_sched_t, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sqrt_inplace'); + late final _ggml_sqrt_inplace = _ggml_sqrt_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - bool ggml_backend_sched_alloc_graph( - ggml_backend_sched_t sched, - ffi.Pointer graph, + ffi.Pointer ggml_ssm_conv( + ffi.Pointer ctx, + ffi.Pointer sx, + ffi.Pointer c, ) { - return _ggml_backend_sched_alloc_graph( - sched, - graph, + return _ggml_ssm_conv( + ctx, + sx, + c, ); } - late final _ggml_backend_sched_alloc_graphPtr = _lookup< + late final _ggml_ssm_convPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ggml_backend_sched_t, - ffi.Pointer)>>('ggml_backend_sched_alloc_graph'); - late final _ggml_backend_sched_alloc_graph = - _ggml_backend_sched_alloc_graphPtr.asFunction< - bool Function(ggml_backend_sched_t, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_ssm_conv'); + late final _ggml_ssm_conv = _ggml_ssm_convPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ggml_status ggml_backend_sched_graph_compute( - ggml_backend_sched_t sched, - ffi.Pointer graph, + ffi.Pointer ggml_ssm_scan( + ffi.Pointer ctx, + ffi.Pointer s, + ffi.Pointer x, + ffi.Pointer dt, + ffi.Pointer A, + ffi.Pointer B, + ffi.Pointer C, + ffi.Pointer ids, ) { - return ggml_status.fromValue(_ggml_backend_sched_graph_compute( - sched, - graph, - )); + return _ggml_ssm_scan( + ctx, + s, + x, + dt, + A, + B, + C, + ids, + ); } - late final _ggml_backend_sched_graph_computePtr = _lookup< + late final _ggml_ssm_scanPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ggml_backend_sched_t, - ffi.Pointer)>>('ggml_backend_sched_graph_compute'); - late final _ggml_backend_sched_graph_compute = - _ggml_backend_sched_graph_computePtr.asFunction< - int Function(ggml_backend_sched_t, ffi.Pointer)>(); - - ggml_status ggml_backend_sched_graph_compute_async( - ggml_backend_sched_t sched, - ffi.Pointer graph, - ) { - return ggml_status.fromValue(_ggml_backend_sched_graph_compute_async( - sched, - graph, - )); - } - - late final _ggml_backend_sched_graph_compute_asyncPtr = _lookup< - ffi.NativeFunction< - ffi.Int Function( - ggml_backend_sched_t, ffi.Pointer)>>( - 'ggml_backend_sched_graph_compute_async'); - late final _ggml_backend_sched_graph_compute_async = - _ggml_backend_sched_graph_compute_asyncPtr.asFunction< - int Function(ggml_backend_sched_t, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_ssm_scan'); + late final _ggml_ssm_scan = _ggml_ssm_scanPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - void ggml_backend_sched_synchronize( - ggml_backend_sched_t sched, + ffi.Pointer ggml_status_to_string( + ggml_status status, ) { - return _ggml_backend_sched_synchronize( - sched, + return _ggml_status_to_string( + status.value, ); } - late final _ggml_backend_sched_synchronizePtr = - _lookup>( - 'ggml_backend_sched_synchronize'); - late final _ggml_backend_sched_synchronize = - _ggml_backend_sched_synchronizePtr - .asFunction(); + late final _ggml_status_to_stringPtr = + _lookup Function(ffi.Int)>>( + 'ggml_status_to_string'); + late final _ggml_status_to_string = _ggml_status_to_stringPtr + .asFunction Function(int)>(); - void ggml_backend_sched_reset( - ggml_backend_sched_t sched, + ffi.Pointer ggml_step( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_reset( - sched, + return _ggml_step( + ctx, + a, ); } - late final _ggml_backend_sched_resetPtr = - _lookup>( - 'ggml_backend_sched_reset'); - late final _ggml_backend_sched_reset = _ggml_backend_sched_resetPtr - .asFunction(); + late final _ggml_stepPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_step'); + late final _ggml_step = _ggml_stepPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_sched_set_eval_callback( - ggml_backend_sched_t sched, - ggml_backend_sched_eval_callback callback, - ffi.Pointer user_data, + ffi.Pointer ggml_step_inplace( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_sched_set_eval_callback( - sched, - callback, - user_data, + return _ggml_step_inplace( + ctx, + a, ); } - late final _ggml_backend_sched_set_eval_callbackPtr = _lookup< + late final _ggml_step_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_backend_sched_t, - ggml_backend_sched_eval_callback, - ffi.Pointer)>>('ggml_backend_sched_set_eval_callback'); - late final _ggml_backend_sched_set_eval_callback = - _ggml_backend_sched_set_eval_callbackPtr.asFunction< - void Function(ggml_backend_sched_t, ggml_backend_sched_eval_callback, - ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_step_inplace'); + late final _ggml_step_inplace = _ggml_step_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_backend_meta_split_axis_name( - ggml_backend_meta_split_axis split_axis, + ffi.Pointer ggml_sub( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_backend_meta_split_axis_name( - split_axis.value, + return _ggml_sub( + ctx, + a, + b, ); } - late final _ggml_backend_meta_split_axis_namePtr = _lookup< - ffi.NativeFunction Function(ffi.UnsignedInt)>>( - 'ggml_backend_meta_split_axis_name'); - late final _ggml_backend_meta_split_axis_name = - _ggml_backend_meta_split_axis_namePtr - .asFunction Function(int)>(); + late final _ggml_subPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>>('ggml_sub'); + late final _ggml_sub = _ggml_subPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_dev_t ggml_backend_meta_device( - ffi.Pointer devs, - int n_devs, - ggml_backend_meta_get_split_state_t get_split_state, - ffi.Pointer get_split_state_ud, + ffi.Pointer ggml_sub_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return _ggml_backend_meta_device( - devs, - n_devs, - get_split_state, - get_split_state_ud, + return _ggml_sub_inplace( + ctx, + a, + b, ); } - late final _ggml_backend_meta_devicePtr = _lookup< + late final _ggml_sub_inplacePtr = _lookup< ffi.NativeFunction< - ggml_backend_dev_t Function( - ffi.Pointer, - ffi.Size, - ggml_backend_meta_get_split_state_t, - ffi.Pointer)>>('ggml_backend_meta_device'); - late final _ggml_backend_meta_device = - _ggml_backend_meta_devicePtr.asFunction< - ggml_backend_dev_t Function(ffi.Pointer, int, - ggml_backend_meta_get_split_state_t, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_sub_inplace'); + late final _ggml_sub_inplace = _ggml_sub_inplacePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_graph_copy$1 ggml_backend_graph_copy( - ggml_backend_t backend, - ffi.Pointer graph, + ffi.Pointer ggml_sum( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_graph_copy( - backend, - graph, + return _ggml_sum( + ctx, + a, ); } - late final _ggml_backend_graph_copyPtr = _lookup< + late final _ggml_sumPtr = _lookup< ffi.NativeFunction< - ggml_backend_graph_copy$1 Function(ggml_backend_t, - ffi.Pointer)>>('ggml_backend_graph_copy'); - late final _ggml_backend_graph_copy = _ggml_backend_graph_copyPtr.asFunction< - ggml_backend_graph_copy$1 Function( - ggml_backend_t, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sum'); + late final _ggml_sum = _ggml_sumPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_graph_copy_free( - ggml_backend_graph_copy$1 copy, + ffi.Pointer ggml_sum_rows( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_graph_copy_free( - copy, + return _ggml_sum_rows( + ctx, + a, ); } - late final _ggml_backend_graph_copy_freePtr = - _lookup>( - 'ggml_backend_graph_copy_free'); - late final _ggml_backend_graph_copy_free = _ggml_backend_graph_copy_freePtr - .asFunction(); + late final _ggml_sum_rowsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_sum_rows'); + late final _ggml_sum_rows = _ggml_sum_rowsPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - bool ggml_backend_compare_graph_backend( - ggml_backend_t backend1, - ggml_backend_t backend2, - ffi.Pointer graph, - ggml_backend_eval_callback callback, - ffi.Pointer user_data, - ffi.Pointer> test_nodes, - int num_test_nodes, + ffi.Pointer ggml_swiglu( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_compare_graph_backend( - backend1, - backend2, - graph, - callback, - user_data, - test_nodes, - num_test_nodes, + return _ggml_swiglu( + ctx, + a, ); } - late final _ggml_backend_compare_graph_backendPtr = _lookup< + late final _ggml_swigluPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function( - ggml_backend_t, - ggml_backend_t, - ffi.Pointer, - ggml_backend_eval_callback, - ffi.Pointer, - ffi.Pointer>, - ffi.Size)>>('ggml_backend_compare_graph_backend'); - late final _ggml_backend_compare_graph_backend = - _ggml_backend_compare_graph_backendPtr.asFunction< - bool Function( - ggml_backend_t, - ggml_backend_t, - ffi.Pointer, - ggml_backend_eval_callback, - ffi.Pointer, - ffi.Pointer>, - int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_swiglu'); + late final _ggml_swiglu = _ggml_swigluPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_status ggml_backend_tensor_alloc( - ggml_backend_buffer_t buffer, - ffi.Pointer tensor, - ffi.Pointer addr, - ) { - return ggml_status.fromValue(_ggml_backend_tensor_alloc( - buffer, - tensor, - addr, - )); + ffi.Pointer ggml_swiglu_oai( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, + double alpha, + double limit, + ) { + return _ggml_swiglu_oai( + ctx, + a, + b, + alpha, + limit, + ); } - late final _ggml_backend_tensor_allocPtr = _lookup< + late final _ggml_swiglu_oaiPtr = _lookup< ffi.NativeFunction< - ffi.Int Function(ggml_backend_buffer_t, ffi.Pointer, - ffi.Pointer)>>('ggml_backend_tensor_alloc'); - late final _ggml_backend_tensor_alloc = - _ggml_backend_tensor_allocPtr.asFunction< - int Function(ggml_backend_buffer_t, ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float)>>('ggml_swiglu_oai'); + late final _ggml_swiglu_oai = _ggml_swiglu_oaiPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + double, + double)>(); - ggml_status ggml_backend_view_init( - ffi.Pointer tensor, + ffi.Pointer ggml_swiglu_split( + ffi.Pointer ctx, + ffi.Pointer a, + ffi.Pointer b, ) { - return ggml_status.fromValue(_ggml_backend_view_init( - tensor, - )); + return _ggml_swiglu_split( + ctx, + a, + b, + ); } - late final _ggml_backend_view_initPtr = - _lookup)>>( - 'ggml_backend_view_init'); - late final _ggml_backend_view_init = _ggml_backend_view_initPtr - .asFunction)>(); + late final _ggml_swiglu_splitPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_swiglu_split'); + late final _ggml_swiglu_split = _ggml_swiglu_splitPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr( - ffi.Pointer ptr, - int size, + ffi.Pointer ggml_swiglu_swapped( + ffi.Pointer ctx, + ffi.Pointer a, ) { - return _ggml_backend_cpu_buffer_from_ptr( - ptr, - size, + return _ggml_swiglu_swapped( + ctx, + a, ); } - late final _ggml_backend_cpu_buffer_from_ptrPtr = _lookup< + late final _ggml_swiglu_swappedPtr = _lookup< ffi.NativeFunction< - ggml_backend_buffer_t Function(ffi.Pointer, - ffi.Size)>>('ggml_backend_cpu_buffer_from_ptr'); - late final _ggml_backend_cpu_buffer_from_ptr = - _ggml_backend_cpu_buffer_from_ptrPtr.asFunction< - ggml_backend_buffer_t Function(ffi.Pointer, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_swiglu_swapped'); + late final _ggml_swiglu_swapped = _ggml_swiglu_swappedPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type() { - return _ggml_backend_cpu_buffer_type(); + ggml_status ggml_tallocr_alloc( + ffi.Pointer talloc, + ffi.Pointer tensor, + ) { + return ggml_status.fromValue(_ggml_tallocr_alloc( + talloc, + tensor, + )); } - late final _ggml_backend_cpu_buffer_typePtr = - _lookup>( - 'ggml_backend_cpu_buffer_type'); - late final _ggml_backend_cpu_buffer_type = _ggml_backend_cpu_buffer_typePtr - .asFunction(); + late final _ggml_tallocr_allocPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, + ffi.Pointer)>>('ggml_tallocr_alloc'); + late final _ggml_tallocr_alloc = _ggml_tallocr_allocPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - void ggml_numa_init( - ggml_numa_strategy numa, + ggml_tallocr ggml_tallocr_new( + ggml_backend_buffer_t buffer, ) { - return _ggml_numa_init( - numa.value, + return _ggml_tallocr_new( + buffer, ); } - late final _ggml_numa_initPtr = - _lookup>( - 'ggml_numa_init'); - late final _ggml_numa_init = - _ggml_numa_initPtr.asFunction(); - - bool ggml_is_numa() { - return _ggml_is_numa(); - } - - late final _ggml_is_numaPtr = - _lookup>('ggml_is_numa'); - late final _ggml_is_numa = _ggml_is_numaPtr.asFunction(); + late final _ggml_tallocr_newPtr = + _lookup>( + 'ggml_tallocr_new'); + late final _ggml_tallocr_new = _ggml_tallocr_newPtr + .asFunction(); - ffi.Pointer ggml_new_i32( + ffi.Pointer ggml_tanh( ffi.Pointer ctx, - int value, + ffi.Pointer a, ) { - return _ggml_new_i32( + return _ggml_tanh( ctx, - value, + a, ); } - late final _ggml_new_i32Ptr = _lookup< + late final _ggml_tanhPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int32)>>('ggml_new_i32'); - late final _ggml_new_i32 = _ggml_new_i32Ptr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_tanh'); + late final _ggml_tanh = _ggml_tanhPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_new_f32( + ffi.Pointer ggml_tanh_inplace( ffi.Pointer ctx, - double value, + ffi.Pointer a, ) { - return _ggml_new_f32( + return _ggml_tanh_inplace( ctx, - value, + a, ); } - late final _ggml_new_f32Ptr = _lookup< + late final _ggml_tanh_inplacePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Float)>>('ggml_new_f32'); - late final _ggml_new_f32 = _ggml_new_f32Ptr.asFunction< - ffi.Pointer Function(ffi.Pointer, double)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_tanh_inplace'); + late final _ggml_tanh_inplace = _ggml_tanh_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer ggml_set_i32( - ffi.Pointer tensor, - int value, - ) { - return _ggml_set_i32( - tensor, - value, - ); + int ggml_tensor_overhead() { + return _ggml_tensor_overhead(); } - late final _ggml_set_i32Ptr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int32)>>('ggml_set_i32'); - late final _ggml_set_i32 = _ggml_set_i32Ptr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + late final _ggml_tensor_overheadPtr = + _lookup>('ggml_tensor_overhead'); + late final _ggml_tensor_overhead = + _ggml_tensor_overheadPtr.asFunction(); - ffi.Pointer ggml_set_f32( - ffi.Pointer tensor, - double value, + void ggml_threadpool_free( + ffi.Pointer threadpool, ) { - return _ggml_set_f32( - tensor, - value, + return _ggml_threadpool_free( + threadpool, ); } - late final _ggml_set_f32Ptr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Float)>>('ggml_set_f32'); - late final _ggml_set_f32 = _ggml_set_f32Ptr.asFunction< - ffi.Pointer Function(ffi.Pointer, double)>(); + late final _ggml_threadpool_freePtr = _lookup< + ffi.NativeFunction)>>( + 'ggml_threadpool_free'); + late final _ggml_threadpool_free = _ggml_threadpool_freePtr + .asFunction)>(); - int ggml_get_i32_1d( - ffi.Pointer tensor, - int i, + int ggml_threadpool_get_n_threads( + ffi.Pointer threadpool, ) { - return _ggml_get_i32_1d( - tensor, - i, + return _ggml_threadpool_get_n_threads( + threadpool, ); } - late final _ggml_get_i32_1dPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, ffi.Int)>>('ggml_get_i32_1d'); - late final _ggml_get_i32_1d = _ggml_get_i32_1dPtr - .asFunction, int)>(); + late final _ggml_threadpool_get_n_threadsPtr = _lookup< + ffi.NativeFunction)>>( + 'ggml_threadpool_get_n_threads'); + late final _ggml_threadpool_get_n_threads = _ggml_threadpool_get_n_threadsPtr + .asFunction)>(); - void ggml_set_i32_1d( - ffi.Pointer tensor, - int i, - int value, + ffi.Pointer ggml_threadpool_new( + ffi.Pointer params, ) { - return _ggml_set_i32_1d( - tensor, - i, - value, + return _ggml_threadpool_new( + params, ); } - late final _ggml_set_i32_1dPtr = _lookup< + late final _ggml_threadpool_newPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Int, - ffi.Int32)>>('ggml_set_i32_1d'); - late final _ggml_set_i32_1d = _ggml_set_i32_1dPtr - .asFunction, int, int)>(); - - int ggml_get_i32_nd( - ffi.Pointer tensor, - int i0, - int i1, - int i2, - int i3, - ) { - return _ggml_get_i32_nd( - tensor, - i0, - i1, - i2, - i3, - ); - } - - late final _ggml_get_i32_ndPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function(ffi.Pointer, ffi.Int, ffi.Int, - ffi.Int, ffi.Int)>>('ggml_get_i32_nd'); - late final _ggml_get_i32_nd = _ggml_get_i32_ndPtr - .asFunction, int, int, int, int)>(); - - void ggml_set_i32_nd( - ffi.Pointer tensor, - int i0, - int i1, - int i2, - int i3, - int value, - ) { - return _ggml_set_i32_nd( - tensor, - i0, - i1, - i2, - i3, - value, - ); - } - - late final _ggml_set_i32_ndPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Int, ffi.Int, ffi.Int, - ffi.Int, ffi.Int32)>>('ggml_set_i32_nd'); - late final _ggml_set_i32_nd = _ggml_set_i32_ndPtr.asFunction< - void Function(ffi.Pointer, int, int, int, int, int)>(); - - double ggml_get_f32_1d( - ffi.Pointer tensor, - int i, - ) { - return _ggml_get_f32_1d( - tensor, - i, - ); - } - - late final _ggml_get_f32_1dPtr = _lookup< - ffi.NativeFunction< - ffi.Float Function( - ffi.Pointer, ffi.Int)>>('ggml_get_f32_1d'); - late final _ggml_get_f32_1d = _ggml_get_f32_1dPtr - .asFunction, int)>(); - - void ggml_set_f32_1d( - ffi.Pointer tensor, - int i, - double value, - ) { - return _ggml_set_f32_1d( - tensor, - i, - value, - ); - } - - late final _ggml_set_f32_1dPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Int, - ffi.Float)>>('ggml_set_f32_1d'); - late final _ggml_set_f32_1d = _ggml_set_f32_1dPtr - .asFunction, int, double)>(); - - double ggml_get_f32_nd( - ffi.Pointer tensor, - int i0, - int i1, - int i2, - int i3, - ) { - return _ggml_get_f32_nd( - tensor, - i0, - i1, - i2, - i3, - ); - } - - late final _ggml_get_f32_ndPtr = _lookup< - ffi.NativeFunction< - ffi.Float Function(ffi.Pointer, ffi.Int, ffi.Int, - ffi.Int, ffi.Int)>>('ggml_get_f32_nd'); - late final _ggml_get_f32_nd = _ggml_get_f32_ndPtr.asFunction< - double Function(ffi.Pointer, int, int, int, int)>(); + ffi.Pointer Function( + ffi.Pointer)>>('ggml_threadpool_new'); + late final _ggml_threadpool_new = _ggml_threadpool_newPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer)>(); - void ggml_set_f32_nd( - ffi.Pointer tensor, - int i0, - int i1, - int i2, - int i3, - double value, + ggml_threadpool_params ggml_threadpool_params_default( + int n_threads, ) { - return _ggml_set_f32_nd( - tensor, - i0, - i1, - i2, - i3, - value, + return _ggml_threadpool_params_default( + n_threads, ); } - late final _ggml_set_f32_ndPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Int, ffi.Int, ffi.Int, - ffi.Int, ffi.Float)>>('ggml_set_f32_nd'); - late final _ggml_set_f32_nd = _ggml_set_f32_ndPtr.asFunction< - void Function(ffi.Pointer, int, int, int, int, double)>(); + late final _ggml_threadpool_params_defaultPtr = + _lookup>( + 'ggml_threadpool_params_default'); + late final _ggml_threadpool_params_default = + _ggml_threadpool_params_defaultPtr + .asFunction(); - ffi.Pointer ggml_threadpool_new( - ffi.Pointer params, + void ggml_threadpool_params_init( + ffi.Pointer p, + int n_threads, ) { - return _ggml_threadpool_new( - params, + return _ggml_threadpool_params_init( + p, + n_threads, ); } - late final _ggml_threadpool_newPtr = _lookup< + late final _ggml_threadpool_params_initPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('ggml_threadpool_new'); - late final _ggml_threadpool_new = _ggml_threadpool_newPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer)>(); - - void ggml_threadpool_free( - ffi.Pointer threadpool, - ) { - return _ggml_threadpool_free( - threadpool, - ); - } - - late final _ggml_threadpool_freePtr = _lookup< - ffi.NativeFunction)>>( - 'ggml_threadpool_free'); - late final _ggml_threadpool_free = _ggml_threadpool_freePtr - .asFunction)>(); + ffi.Void Function(ffi.Pointer, + ffi.Int)>>('ggml_threadpool_params_init'); + late final _ggml_threadpool_params_init = _ggml_threadpool_params_initPtr + .asFunction, int)>(); - int ggml_threadpool_get_n_threads( - ffi.Pointer threadpool, + bool ggml_threadpool_params_match( + ffi.Pointer p0, + ffi.Pointer p1, ) { - return _ggml_threadpool_get_n_threads( - threadpool, + return _ggml_threadpool_params_match( + p0, + p1, ); } - late final _ggml_threadpool_get_n_threadsPtr = _lookup< - ffi.NativeFunction)>>( - 'ggml_threadpool_get_n_threads'); - late final _ggml_threadpool_get_n_threads = _ggml_threadpool_get_n_threadsPtr - .asFunction)>(); + late final _ggml_threadpool_params_matchPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, + ffi.Pointer)>>( + 'ggml_threadpool_params_match'); + late final _ggml_threadpool_params_match = + _ggml_threadpool_params_matchPtr.asFunction< + bool Function(ffi.Pointer, + ffi.Pointer)>(); void ggml_threadpool_pause( ffi.Pointer threadpool, @@ -12402,2486 +12587,2508 @@ class LlamaBindings { late final _ggml_threadpool_resume = _ggml_threadpool_resumePtr .asFunction)>(); - ggml_cplan ggml_graph_plan( - ffi.Pointer cgraph, - int n_threads, - ffi.Pointer threadpool, - ) { - return _ggml_graph_plan( - cgraph, - n_threads, - threadpool, - ); + void ggml_time_init() { + return _ggml_time_init(); } - late final _ggml_graph_planPtr = _lookup< - ffi.NativeFunction< - ggml_cplan Function(ffi.Pointer, ffi.Int, - ffi.Pointer)>>('ggml_graph_plan'); - late final _ggml_graph_plan = _ggml_graph_planPtr.asFunction< - ggml_cplan Function( - ffi.Pointer, int, ffi.Pointer)>(); + late final _ggml_time_initPtr = + _lookup>('ggml_time_init'); + late final _ggml_time_init = _ggml_time_initPtr.asFunction(); - ggml_status ggml_graph_compute( - ffi.Pointer cgraph, - ffi.Pointer cplan, - ) { - return ggml_status.fromValue(_ggml_graph_compute( - cgraph, - cplan, - )); + int ggml_time_ms() { + return _ggml_time_ms(); } - late final _ggml_graph_computePtr = _lookup< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, - ffi.Pointer)>>('ggml_graph_compute'); - late final _ggml_graph_compute = _ggml_graph_computePtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer)>(); + late final _ggml_time_msPtr = + _lookup>('ggml_time_ms'); + late final _ggml_time_ms = _ggml_time_msPtr.asFunction(); - ggml_status ggml_graph_compute_with_ctx( - ffi.Pointer ctx, - ffi.Pointer cgraph, - int n_threads, - ) { - return ggml_status.fromValue(_ggml_graph_compute_with_ctx( - ctx, - cgraph, - n_threads, - )); + int ggml_time_us() { + return _ggml_time_us(); } - late final _ggml_graph_compute_with_ctxPtr = _lookup< - ffi.NativeFunction< - ffi.Int Function(ffi.Pointer, ffi.Pointer, - ffi.Int)>>('ggml_graph_compute_with_ctx'); - late final _ggml_graph_compute_with_ctx = - _ggml_graph_compute_with_ctxPtr.asFunction< - int Function( - ffi.Pointer, ffi.Pointer, int)>(); + late final _ggml_time_usPtr = + _lookup>('ggml_time_us'); + late final _ggml_time_us = _ggml_time_usPtr.asFunction(); - int ggml_cpu_has_sse3() { - return _ggml_cpu_has_sse3(); + ffi.Pointer ggml_timestep_embedding( + ffi.Pointer ctx, + ffi.Pointer timesteps, + int dim, + int max_period, + ) { + return _ggml_timestep_embedding( + ctx, + timesteps, + dim, + max_period, + ); } - late final _ggml_cpu_has_sse3Ptr = - _lookup>('ggml_cpu_has_sse3'); - late final _ggml_cpu_has_sse3 = - _ggml_cpu_has_sse3Ptr.asFunction(); + late final _ggml_timestep_embeddingPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int)>>('ggml_timestep_embedding'); + late final _ggml_timestep_embedding = _ggml_timestep_embeddingPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - int ggml_cpu_has_ssse3() { - return _ggml_cpu_has_ssse3(); + ffi.Pointer ggml_top_k( + ffi.Pointer ctx, + ffi.Pointer a, + int k, + ) { + return _ggml_top_k( + ctx, + a, + k, + ); } - late final _ggml_cpu_has_ssse3Ptr = - _lookup>('ggml_cpu_has_ssse3'); - late final _ggml_cpu_has_ssse3 = - _ggml_cpu_has_ssse3Ptr.asFunction(); + late final _ggml_top_kPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Int)>>('ggml_top_k'); + late final _ggml_top_k = _ggml_top_kPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - int ggml_cpu_has_avx() { - return _ggml_cpu_has_avx(); + ffi.Pointer ggml_transpose( + ffi.Pointer ctx, + ffi.Pointer a, + ) { + return _ggml_transpose( + ctx, + a, + ); } - late final _ggml_cpu_has_avxPtr = - _lookup>('ggml_cpu_has_avx'); - late final _ggml_cpu_has_avx = - _ggml_cpu_has_avxPtr.asFunction(); + late final _ggml_transposePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_transpose'); + late final _ggml_transpose = _ggml_transposePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - int ggml_cpu_has_avx_vnni() { - return _ggml_cpu_has_avx_vnni(); + ffi.Pointer ggml_tri( + ffi.Pointer ctx, + ffi.Pointer a, + ggml_tri_type type, + ) { + return _ggml_tri( + ctx, + a, + type.value, + ); } - late final _ggml_cpu_has_avx_vnniPtr = - _lookup>('ggml_cpu_has_avx_vnni'); - late final _ggml_cpu_has_avx_vnni = - _ggml_cpu_has_avx_vnniPtr.asFunction(); + late final _ggml_triPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.UnsignedInt)>>('ggml_tri'); + late final _ggml_tri = _ggml_triPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - int ggml_cpu_has_avx2() { - return _ggml_cpu_has_avx2(); + /// Truncates the fractional part of each element in the tensor (towards zero). + /// For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0 + /// Similar to std::trunc in C/C++. + ffi.Pointer ggml_trunc( + ffi.Pointer ctx, + ffi.Pointer a, + ) { + return _ggml_trunc( + ctx, + a, + ); } - late final _ggml_cpu_has_avx2Ptr = - _lookup>('ggml_cpu_has_avx2'); - late final _ggml_cpu_has_avx2 = - _ggml_cpu_has_avx2Ptr.asFunction(); + late final _ggml_truncPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_trunc'); + late final _ggml_trunc = _ggml_truncPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - int ggml_cpu_has_bmi2() { - return _ggml_cpu_has_bmi2(); + ffi.Pointer ggml_trunc_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ) { + return _ggml_trunc_inplace( + ctx, + a, + ); } - late final _ggml_cpu_has_bmi2Ptr = - _lookup>('ggml_cpu_has_bmi2'); - late final _ggml_cpu_has_bmi2 = - _ggml_cpu_has_bmi2Ptr.asFunction(); + late final _ggml_trunc_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_trunc_inplace'); + late final _ggml_trunc_inplace = _ggml_trunc_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - int ggml_cpu_has_f16c() { - return _ggml_cpu_has_f16c(); + ffi.Pointer ggml_type_name( + ggml_type type, + ) { + return _ggml_type_name( + type.value, + ); } - late final _ggml_cpu_has_f16cPtr = - _lookup>('ggml_cpu_has_f16c'); - late final _ggml_cpu_has_f16c = - _ggml_cpu_has_f16cPtr.asFunction(); + late final _ggml_type_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'ggml_type_name'); + late final _ggml_type_name = + _ggml_type_namePtr.asFunction Function(int)>(); - int ggml_cpu_has_fma() { - return _ggml_cpu_has_fma(); + int ggml_type_size( + ggml_type type, + ) { + return _ggml_type_size( + type.value, + ); } - late final _ggml_cpu_has_fmaPtr = - _lookup>('ggml_cpu_has_fma'); - late final _ggml_cpu_has_fma = - _ggml_cpu_has_fmaPtr.asFunction(); + late final _ggml_type_sizePtr = + _lookup>( + 'ggml_type_size'); + late final _ggml_type_size = + _ggml_type_sizePtr.asFunction(); - int ggml_cpu_has_avx512() { - return _ggml_cpu_has_avx512(); + @Deprecated('use ggml_row_size() instead') + double ggml_type_sizef( + ggml_type type, + ) { + return _ggml_type_sizef( + type.value, + ); } - late final _ggml_cpu_has_avx512Ptr = - _lookup>('ggml_cpu_has_avx512'); - late final _ggml_cpu_has_avx512 = - _ggml_cpu_has_avx512Ptr.asFunction(); + late final _ggml_type_sizefPtr = + _lookup>( + 'ggml_type_sizef'); + late final _ggml_type_sizef = + _ggml_type_sizefPtr.asFunction(); - int ggml_cpu_has_avx512_vbmi() { - return _ggml_cpu_has_avx512_vbmi(); + ffi.Pointer ggml_unary( + ffi.Pointer ctx, + ffi.Pointer a, + ggml_unary_op op, + ) { + return _ggml_unary( + ctx, + a, + op.value, + ); } - late final _ggml_cpu_has_avx512_vbmiPtr = - _lookup>( - 'ggml_cpu_has_avx512_vbmi'); - late final _ggml_cpu_has_avx512_vbmi = - _ggml_cpu_has_avx512_vbmiPtr.asFunction(); + late final _ggml_unaryPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.UnsignedInt)>>('ggml_unary'); + late final _ggml_unary = _ggml_unaryPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - int ggml_cpu_has_avx512_vnni() { - return _ggml_cpu_has_avx512_vnni(); + ffi.Pointer ggml_unary_inplace( + ffi.Pointer ctx, + ffi.Pointer a, + ggml_unary_op op, + ) { + return _ggml_unary_inplace( + ctx, + a, + op.value, + ); } - late final _ggml_cpu_has_avx512_vnniPtr = - _lookup>( - 'ggml_cpu_has_avx512_vnni'); - late final _ggml_cpu_has_avx512_vnni = - _ggml_cpu_has_avx512_vnniPtr.asFunction(); - - int ggml_cpu_has_avx512_bf16() { - return _ggml_cpu_has_avx512_bf16(); - } + late final _ggml_unary_inplacePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.UnsignedInt)>>('ggml_unary_inplace'); + late final _ggml_unary_inplace = _ggml_unary_inplacePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - late final _ggml_cpu_has_avx512_bf16Ptr = - _lookup>( - 'ggml_cpu_has_avx512_bf16'); - late final _ggml_cpu_has_avx512_bf16 = - _ggml_cpu_has_avx512_bf16Ptr.asFunction(); - - int ggml_cpu_has_amx_int8() { - return _ggml_cpu_has_amx_int8(); - } - - late final _ggml_cpu_has_amx_int8Ptr = - _lookup>('ggml_cpu_has_amx_int8'); - late final _ggml_cpu_has_amx_int8 = - _ggml_cpu_has_amx_int8Ptr.asFunction(); - - int ggml_cpu_has_neon() { - return _ggml_cpu_has_neon(); - } - - late final _ggml_cpu_has_neonPtr = - _lookup>('ggml_cpu_has_neon'); - late final _ggml_cpu_has_neon = - _ggml_cpu_has_neonPtr.asFunction(); - - int ggml_cpu_has_arm_fma() { - return _ggml_cpu_has_arm_fma(); + ffi.Pointer ggml_unary_op_name( + ggml_unary_op op, + ) { + return _ggml_unary_op_name( + op.value, + ); } - late final _ggml_cpu_has_arm_fmaPtr = - _lookup>('ggml_cpu_has_arm_fma'); - late final _ggml_cpu_has_arm_fma = - _ggml_cpu_has_arm_fmaPtr.asFunction(); + late final _ggml_unary_op_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'ggml_unary_op_name'); + late final _ggml_unary_op_name = + _ggml_unary_op_namePtr.asFunction Function(int)>(); - int ggml_cpu_has_fp16_va() { - return _ggml_cpu_has_fp16_va(); + void ggml_unravel_index( + ffi.Pointer tensor, + int i, + ffi.Pointer i0, + ffi.Pointer i1, + ffi.Pointer i2, + ffi.Pointer i3, + ) { + return _ggml_unravel_index( + tensor, + i, + i0, + i1, + i2, + i3, + ); } - late final _ggml_cpu_has_fp16_vaPtr = - _lookup>('ggml_cpu_has_fp16_va'); - late final _ggml_cpu_has_fp16_va = - _ggml_cpu_has_fp16_vaPtr.asFunction(); + late final _ggml_unravel_indexPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, + ffi.Int64, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('ggml_unravel_index'); + late final _ggml_unravel_index = _ggml_unravel_indexPtr.asFunction< + void Function( + ffi.Pointer, + int, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>(); - int ggml_cpu_has_dotprod() { - return _ggml_cpu_has_dotprod(); + ffi.Pointer ggml_upscale( + ffi.Pointer ctx, + ffi.Pointer a, + int scale_factor, + ggml_scale_mode mode, + ) { + return _ggml_upscale( + ctx, + a, + scale_factor, + mode.value, + ); } - late final _ggml_cpu_has_dotprodPtr = - _lookup>('ggml_cpu_has_dotprod'); - late final _ggml_cpu_has_dotprod = - _ggml_cpu_has_dotprodPtr.asFunction(); + late final _ggml_upscalePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.UnsignedInt)>>('ggml_upscale'); + late final _ggml_upscale = _ggml_upscalePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - int ggml_cpu_has_matmul_int8() { - return _ggml_cpu_has_matmul_int8(); + @Deprecated('use ggml_interpolate instead') + ffi.Pointer ggml_upscale_ext( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int ne1, + int ne2, + int ne3, + ggml_scale_mode mode, + ) { + return _ggml_upscale_ext( + ctx, + a, + ne0, + ne1, + ne2, + ne3, + mode.value, + ); } - late final _ggml_cpu_has_matmul_int8Ptr = - _lookup>( - 'ggml_cpu_has_matmul_int8'); - late final _ggml_cpu_has_matmul_int8 = - _ggml_cpu_has_matmul_int8Ptr.asFunction(); + late final _ggml_upscale_extPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.Int, + ffi.UnsignedInt)>>('ggml_upscale_ext'); + late final _ggml_upscale_ext = _ggml_upscale_extPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int, int)>(); - int ggml_cpu_has_sve() { - return _ggml_cpu_has_sve(); + int ggml_used_mem( + ffi.Pointer ctx, + ) { + return _ggml_used_mem( + ctx, + ); } - late final _ggml_cpu_has_svePtr = - _lookup>('ggml_cpu_has_sve'); - late final _ggml_cpu_has_sve = - _ggml_cpu_has_svePtr.asFunction(); + late final _ggml_used_memPtr = + _lookup)>>( + 'ggml_used_mem'); + late final _ggml_used_mem = + _ggml_used_memPtr.asFunction)>(); - int ggml_cpu_get_sve_cnt() { - return _ggml_cpu_get_sve_cnt(); + bool ggml_validate_row_data( + ggml_type type, + ffi.Pointer data, + int nbytes, + ) { + return _ggml_validate_row_data( + type.value, + data, + nbytes, + ); } - late final _ggml_cpu_get_sve_cntPtr = - _lookup>('ggml_cpu_get_sve_cnt'); - late final _ggml_cpu_get_sve_cnt = - _ggml_cpu_get_sve_cntPtr.asFunction(); + late final _ggml_validate_row_dataPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.UnsignedInt, ffi.Pointer, + ffi.Size)>>('ggml_validate_row_data'); + late final _ggml_validate_row_data = _ggml_validate_row_dataPtr + .asFunction, int)>(); - int ggml_cpu_has_sme() { - return _ggml_cpu_has_sme(); + ffi.Pointer ggml_version() { + return _ggml_version(); } - late final _ggml_cpu_has_smePtr = - _lookup>('ggml_cpu_has_sme'); - late final _ggml_cpu_has_sme = - _ggml_cpu_has_smePtr.asFunction(); + late final _ggml_versionPtr = + _lookup Function()>>( + 'ggml_version'); + late final _ggml_version = + _ggml_versionPtr.asFunction Function()>(); - int ggml_cpu_has_riscv_v() { - return _ggml_cpu_has_riscv_v(); + ffi.Pointer ggml_view_1d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int offset, + ) { + return _ggml_view_1d( + ctx, + a, + ne0, + offset, + ); } - late final _ggml_cpu_has_riscv_vPtr = - _lookup>('ggml_cpu_has_riscv_v'); - late final _ggml_cpu_has_riscv_v = - _ggml_cpu_has_riscv_vPtr.asFunction(); + late final _ggml_view_1dPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Int64, ffi.Size)>>('ggml_view_1d'); + late final _ggml_view_1d = _ggml_view_1dPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - int ggml_cpu_get_rvv_vlen() { - return _ggml_cpu_get_rvv_vlen(); + ffi.Pointer ggml_view_2d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int ne1, + int nb1, + int offset, + ) { + return _ggml_view_2d( + ctx, + a, + ne0, + ne1, + nb1, + offset, + ); } - late final _ggml_cpu_get_rvv_vlenPtr = - _lookup>('ggml_cpu_get_rvv_vlen'); - late final _ggml_cpu_get_rvv_vlen = - _ggml_cpu_get_rvv_vlenPtr.asFunction(); + late final _ggml_view_2dPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64, + ffi.Size, + ffi.Size)>>('ggml_view_2d'); + late final _ggml_view_2d = _ggml_view_2dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int)>(); - int ggml_cpu_has_vsx() { - return _ggml_cpu_has_vsx(); - } - - late final _ggml_cpu_has_vsxPtr = - _lookup>('ggml_cpu_has_vsx'); - late final _ggml_cpu_has_vsx = - _ggml_cpu_has_vsxPtr.asFunction(); - - int ggml_cpu_has_vxe() { - return _ggml_cpu_has_vxe(); - } - - late final _ggml_cpu_has_vxePtr = - _lookup>('ggml_cpu_has_vxe'); - late final _ggml_cpu_has_vxe = - _ggml_cpu_has_vxePtr.asFunction(); - - int ggml_cpu_has_wasm_simd() { - return _ggml_cpu_has_wasm_simd(); - } - - late final _ggml_cpu_has_wasm_simdPtr = - _lookup>('ggml_cpu_has_wasm_simd'); - late final _ggml_cpu_has_wasm_simd = - _ggml_cpu_has_wasm_simdPtr.asFunction(); - - int ggml_cpu_has_llamafile() { - return _ggml_cpu_has_llamafile(); - } - - late final _ggml_cpu_has_llamafilePtr = - _lookup>('ggml_cpu_has_llamafile'); - late final _ggml_cpu_has_llamafile = - _ggml_cpu_has_llamafilePtr.asFunction(); - - ffi.Pointer ggml_get_type_traits_cpu( - ggml_type type, + ffi.Pointer ggml_view_3d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int ne1, + int ne2, + int nb1, + int nb2, + int offset, ) { - return _ggml_get_type_traits_cpu( - type.value, + return _ggml_view_3d( + ctx, + a, + ne0, + ne1, + ne2, + nb1, + nb2, + offset, ); } - late final _ggml_get_type_traits_cpuPtr = _lookup< + late final _ggml_view_3dPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.UnsignedInt)>>('ggml_get_type_traits_cpu'); - late final _ggml_get_type_traits_cpu = _ggml_get_type_traits_cpuPtr - .asFunction Function(int)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_view_3d'); + late final _ggml_view_3d = _ggml_view_3dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int, int, int)>(); - void ggml_cpu_init() { - return _ggml_cpu_init(); + ffi.Pointer ggml_view_4d( + ffi.Pointer ctx, + ffi.Pointer a, + int ne0, + int ne1, + int ne2, + int ne3, + int nb1, + int nb2, + int nb3, + int offset, + ) { + return _ggml_view_4d( + ctx, + a, + ne0, + ne1, + ne2, + ne3, + nb1, + nb2, + nb3, + offset, + ); } - late final _ggml_cpu_initPtr = - _lookup>('ggml_cpu_init'); - late final _ggml_cpu_init = _ggml_cpu_initPtr.asFunction(); + late final _ggml_view_4dPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Int64, + ffi.Size, + ffi.Size, + ffi.Size, + ffi.Size)>>('ggml_view_4d'); + late final _ggml_view_4d = _ggml_view_4dPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int, int, int, int, int, int)>(); - ggml_backend_t ggml_backend_cpu_init() { - return _ggml_backend_cpu_init(); + ffi.Pointer ggml_view_tensor( + ffi.Pointer ctx, + ffi.Pointer src, + ) { + return _ggml_view_tensor( + ctx, + src, + ); } - late final _ggml_backend_cpu_initPtr = - _lookup>( - 'ggml_backend_cpu_init'); - late final _ggml_backend_cpu_init = - _ggml_backend_cpu_initPtr.asFunction(); + late final _ggml_view_tensorPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('ggml_view_tensor'); + late final _ggml_view_tensor = _ggml_view_tensorPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - bool ggml_backend_is_cpu( - ggml_backend_t backend, + ffi.Pointer ggml_win_part( + ffi.Pointer ctx, + ffi.Pointer a, + int w, ) { - return _ggml_backend_is_cpu( - backend, + return _ggml_win_part( + ctx, + a, + w, ); } - late final _ggml_backend_is_cpuPtr = - _lookup>( - 'ggml_backend_is_cpu'); - late final _ggml_backend_is_cpu = - _ggml_backend_is_cpuPtr.asFunction(); + late final _ggml_win_partPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Int)>>('ggml_win_part'); + late final _ggml_win_part = _ggml_win_partPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer, int)>(); - void ggml_backend_cpu_set_n_threads( - ggml_backend_t backend_cpu, - int n_threads, + ffi.Pointer ggml_win_unpart( + ffi.Pointer ctx, + ffi.Pointer a, + int w0, + int h0, + int w, ) { - return _ggml_backend_cpu_set_n_threads( - backend_cpu, - n_threads, + return _ggml_win_unpart( + ctx, + a, + w0, + h0, + w, ); } - late final _ggml_backend_cpu_set_n_threadsPtr = - _lookup>( - 'ggml_backend_cpu_set_n_threads'); - late final _ggml_backend_cpu_set_n_threads = - _ggml_backend_cpu_set_n_threadsPtr - .asFunction(); + late final _ggml_win_unpartPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int, + ffi.Int, + ffi.Int)>>('ggml_win_unpart'); + late final _ggml_win_unpart = _ggml_win_unpartPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, int, int, int)>(); - void ggml_backend_cpu_set_threadpool( - ggml_backend_t backend_cpu, - ggml_threadpool_t threadpool, + ffi.Pointer ggml_xielu( + ffi.Pointer ctx, + ffi.Pointer a, + double alpha_n, + double alpha_p, + double beta, + double eps, ) { - return _ggml_backend_cpu_set_threadpool( - backend_cpu, - threadpool, + return _ggml_xielu( + ctx, + a, + alpha_n, + alpha_p, + beta, + eps, ); } - late final _ggml_backend_cpu_set_threadpoolPtr = _lookup< + late final _ggml_xieluPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_t, - ggml_threadpool_t)>>('ggml_backend_cpu_set_threadpool'); - late final _ggml_backend_cpu_set_threadpool = - _ggml_backend_cpu_set_threadpoolPtr - .asFunction(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Float, + ffi.Float, + ffi.Float, + ffi.Float)>>('ggml_xielu'); + late final _ggml_xielu = _ggml_xieluPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, double, double, double, double)>(); - void ggml_backend_cpu_set_abort_callback( - ggml_backend_t backend_cpu, - ggml_abort_callback abort_callback, - ffi.Pointer abort_callback_data, + void gguf_add_tensor( + ffi.Pointer ctx, + ffi.Pointer tensor, ) { - return _ggml_backend_cpu_set_abort_callback( - backend_cpu, - abort_callback, - abort_callback_data, + return _gguf_add_tensor( + ctx, + tensor, ); } - late final _ggml_backend_cpu_set_abort_callbackPtr = _lookup< + late final _gguf_add_tensorPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_backend_t, ggml_abort_callback, - ffi.Pointer)>>('ggml_backend_cpu_set_abort_callback'); - late final _ggml_backend_cpu_set_abort_callback = - _ggml_backend_cpu_set_abort_callbackPtr.asFunction< - void Function( - ggml_backend_t, ggml_abort_callback, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('gguf_add_tensor'); + late final _gguf_add_tensor = _gguf_add_tensorPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - void ggml_backend_cpu_set_use_ref( - ggml_backend_t backend_cpu, - bool use_ref, + int gguf_find_key( + ffi.Pointer ctx, + ffi.Pointer key, ) { - return _ggml_backend_cpu_set_use_ref( - backend_cpu, - use_ref, + return _gguf_find_key( + ctx, + key, ); } - late final _ggml_backend_cpu_set_use_refPtr = - _lookup>( - 'ggml_backend_cpu_set_use_ref'); - late final _ggml_backend_cpu_set_use_ref = _ggml_backend_cpu_set_use_refPtr - .asFunction(); - - ggml_backend_reg_t ggml_backend_cpu_reg() { - return _ggml_backend_cpu_reg(); - } - - late final _ggml_backend_cpu_regPtr = - _lookup>( - 'ggml_backend_cpu_reg'); - late final _ggml_backend_cpu_reg = - _ggml_backend_cpu_regPtr.asFunction(); + late final _gguf_find_keyPtr = _lookup< + ffi.NativeFunction< + ffi.Int64 Function(ffi.Pointer, + ffi.Pointer)>>('gguf_find_key'); + late final _gguf_find_key = _gguf_find_keyPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - void ggml_cpu_fp32_to_fp32( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + int gguf_find_tensor( + ffi.Pointer ctx, + ffi.Pointer name, ) { - return _ggml_cpu_fp32_to_fp32( - arg0, - arg1, - arg2, + return _gguf_find_tensor( + ctx, + name, ); } - late final _ggml_cpu_fp32_to_fp32Ptr = _lookup< + late final _gguf_find_tensorPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_cpu_fp32_to_fp32'); - late final _ggml_cpu_fp32_to_fp32 = _ggml_cpu_fp32_to_fp32Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Int64 Function(ffi.Pointer, + ffi.Pointer)>>('gguf_find_tensor'); + late final _gguf_find_tensor = _gguf_find_tensorPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - void ggml_cpu_fp32_to_i32( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + void gguf_free( + ffi.Pointer ctx, ) { - return _ggml_cpu_fp32_to_i32( - arg0, - arg1, - arg2, + return _gguf_free( + ctx, ); } - late final _ggml_cpu_fp32_to_i32Ptr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_cpu_fp32_to_i32'); - late final _ggml_cpu_fp32_to_i32 = _ggml_cpu_fp32_to_i32Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _gguf_freePtr = + _lookup)>>( + 'gguf_free'); + late final _gguf_free = + _gguf_freePtr.asFunction)>(); - void ggml_cpu_fp32_to_fp16( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + int gguf_get_alignment( + ffi.Pointer ctx, ) { - return _ggml_cpu_fp32_to_fp16( - arg0, - arg1, - arg2, + return _gguf_get_alignment( + ctx, ); } - late final _ggml_cpu_fp32_to_fp16Ptr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_cpu_fp32_to_fp16'); - late final _ggml_cpu_fp32_to_fp16 = _ggml_cpu_fp32_to_fp16Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _gguf_get_alignmentPtr = + _lookup)>>( + 'gguf_get_alignment'); + late final _gguf_get_alignment = _gguf_get_alignmentPtr + .asFunction)>(); - void ggml_cpu_fp16_to_fp32( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + ffi.Pointer gguf_get_arr_data( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_cpu_fp16_to_fp32( - arg0, - arg1, - arg2, + return _gguf_get_arr_data( + ctx, + key_id, ); } - late final _ggml_cpu_fp16_to_fp32Ptr = _lookup< + late final _gguf_get_arr_dataPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_cpu_fp16_to_fp32'); - late final _ggml_cpu_fp16_to_fp32 = _ggml_cpu_fp16_to_fp32Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_arr_data'); + late final _gguf_get_arr_data = _gguf_get_arr_dataPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - void ggml_cpu_fp32_to_bf16( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + int gguf_get_arr_n( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_cpu_fp32_to_bf16( - arg0, - arg1, - arg2, + return _gguf_get_arr_n( + ctx, + key_id, ); } - late final _ggml_cpu_fp32_to_bf16Ptr = _lookup< + late final _gguf_get_arr_nPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_cpu_fp32_to_bf16'); - late final _ggml_cpu_fp32_to_bf16 = _ggml_cpu_fp32_to_bf16Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Size Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_arr_n'); + late final _gguf_get_arr_n = _gguf_get_arr_nPtr + .asFunction, int)>(); - void ggml_cpu_bf16_to_fp32( - ffi.Pointer arg0, - ffi.Pointer arg1, - int arg2, + ffi.Pointer gguf_get_arr_str( + ffi.Pointer ctx, + int key_id, + int i, ) { - return _ggml_cpu_bf16_to_fp32( - arg0, - arg1, - arg2, + return _gguf_get_arr_str( + ctx, + key_id, + i, ); } - late final _ggml_cpu_bf16_to_fp32Ptr = _lookup< + late final _gguf_get_arr_strPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('ggml_cpu_bf16_to_fp32'); - late final _ggml_cpu_bf16_to_fp32 = _ggml_cpu_bf16_to_fp32Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer Function(ffi.Pointer, ffi.Int64, + ffi.Size)>>('gguf_get_arr_str'); + late final _gguf_get_arr_str = _gguf_get_arr_strPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int, int)>(); - ggml_opt_dataset_t ggml_opt_dataset_init( - ggml_type type_data, - ggml_type type_label, - int ne_datapoint, - int ne_label, - int ndata, - int ndata_shard, + gguf_type gguf_get_arr_type( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_dataset_init( - type_data.value, - type_label.value, - ne_datapoint, - ne_label, - ndata, - ndata_shard, - ); + return gguf_type.fromValue(_gguf_get_arr_type( + ctx, + key_id, + )); } - late final _ggml_opt_dataset_initPtr = _lookup< + late final _gguf_get_arr_typePtr = _lookup< ffi.NativeFunction< - ggml_opt_dataset_t Function( - ffi.UnsignedInt, - ffi.UnsignedInt, - ffi.Int64, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_opt_dataset_init'); - late final _ggml_opt_dataset_init = _ggml_opt_dataset_initPtr - .asFunction(); + ffi.UnsignedInt Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_arr_type'); + late final _gguf_get_arr_type = _gguf_get_arr_typePtr + .asFunction, int)>(); - void ggml_opt_dataset_free( - ggml_opt_dataset_t dataset, + int gguf_get_data_offset( + ffi.Pointer ctx, ) { - return _ggml_opt_dataset_free( - dataset, + return _gguf_get_data_offset( + ctx, ); } - late final _ggml_opt_dataset_freePtr = - _lookup>( - 'ggml_opt_dataset_free'); - late final _ggml_opt_dataset_free = - _ggml_opt_dataset_freePtr.asFunction(); + late final _gguf_get_data_offsetPtr = + _lookup)>>( + 'gguf_get_data_offset'); + late final _gguf_get_data_offset = _gguf_get_data_offsetPtr + .asFunction)>(); - int ggml_opt_dataset_ndata( - ggml_opt_dataset_t dataset, + ffi.Pointer gguf_get_key( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_dataset_ndata( - dataset, + return _gguf_get_key( + ctx, + key_id, ); } - late final _ggml_opt_dataset_ndataPtr = - _lookup>( - 'ggml_opt_dataset_ndata'); - late final _ggml_opt_dataset_ndata = - _ggml_opt_dataset_ndataPtr.asFunction(); - - ffi.Pointer ggml_opt_dataset_data( - ggml_opt_dataset_t dataset, + late final _gguf_get_keyPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_key'); + late final _gguf_get_key = _gguf_get_keyPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); + + gguf_type gguf_get_kv_type( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_dataset_data( - dataset, - ); + return gguf_type.fromValue(_gguf_get_kv_type( + ctx, + key_id, + )); } - late final _ggml_opt_dataset_dataPtr = _lookup< + late final _gguf_get_kv_typePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_opt_dataset_t)>>('ggml_opt_dataset_data'); - late final _ggml_opt_dataset_data = _ggml_opt_dataset_dataPtr - .asFunction Function(ggml_opt_dataset_t)>(); + ffi.UnsignedInt Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_kv_type'); + late final _gguf_get_kv_type = _gguf_get_kv_typePtr + .asFunction, int)>(); - ffi.Pointer ggml_opt_dataset_labels( - ggml_opt_dataset_t dataset, + void gguf_get_meta_data( + ffi.Pointer ctx, + ffi.Pointer data, ) { - return _ggml_opt_dataset_labels( - dataset, + return _gguf_get_meta_data( + ctx, + data, ); } - late final _ggml_opt_dataset_labelsPtr = _lookup< + late final _gguf_get_meta_dataPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_opt_dataset_t)>>('ggml_opt_dataset_labels'); - late final _ggml_opt_dataset_labels = _ggml_opt_dataset_labelsPtr - .asFunction Function(ggml_opt_dataset_t)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('gguf_get_meta_data'); + late final _gguf_get_meta_data = _gguf_get_meta_dataPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - void ggml_opt_dataset_shuffle( - ggml_opt_context_t opt_ctx, - ggml_opt_dataset_t dataset, - int idata, + int gguf_get_meta_size( + ffi.Pointer ctx, ) { - return _ggml_opt_dataset_shuffle( - opt_ctx, - dataset, - idata, + return _gguf_get_meta_size( + ctx, ); } - late final _ggml_opt_dataset_shufflePtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ggml_opt_context_t, ggml_opt_dataset_t, - ffi.Int64)>>('ggml_opt_dataset_shuffle'); - late final _ggml_opt_dataset_shuffle = _ggml_opt_dataset_shufflePtr - .asFunction(); + late final _gguf_get_meta_sizePtr = + _lookup)>>( + 'gguf_get_meta_size'); + late final _gguf_get_meta_size = _gguf_get_meta_sizePtr + .asFunction)>(); - void ggml_opt_dataset_get_batch( - ggml_opt_dataset_t dataset, - ffi.Pointer data_batch, - ffi.Pointer labels_batch, - int ibatch, + int gguf_get_n_kv( + ffi.Pointer ctx, ) { - return _ggml_opt_dataset_get_batch( - dataset, - data_batch, - labels_batch, - ibatch, + return _gguf_get_n_kv( + ctx, ); } - late final _ggml_opt_dataset_get_batchPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ggml_opt_dataset_t, - ffi.Pointer, - ffi.Pointer, - ffi.Int64)>>('ggml_opt_dataset_get_batch'); - late final _ggml_opt_dataset_get_batch = - _ggml_opt_dataset_get_batchPtr.asFunction< - void Function(ggml_opt_dataset_t, ffi.Pointer, - ffi.Pointer, int)>(); + late final _gguf_get_n_kvPtr = _lookup< + ffi.NativeFunction)>>( + 'gguf_get_n_kv'); + late final _gguf_get_n_kv = + _gguf_get_n_kvPtr.asFunction)>(); - void ggml_opt_dataset_get_batch_host( - ggml_opt_dataset_t dataset, - ffi.Pointer data_batch, - int nb_data_batch, - ffi.Pointer labels_batch, - int ibatch, + int gguf_get_n_tensors( + ffi.Pointer ctx, ) { - return _ggml_opt_dataset_get_batch_host( - dataset, - data_batch, - nb_data_batch, - labels_batch, - ibatch, + return _gguf_get_n_tensors( + ctx, ); } - late final _ggml_opt_dataset_get_batch_hostPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ggml_opt_dataset_t, - ffi.Pointer, - ffi.Size, - ffi.Pointer, - ffi.Int64)>>('ggml_opt_dataset_get_batch_host'); - late final _ggml_opt_dataset_get_batch_host = - _ggml_opt_dataset_get_batch_hostPtr.asFunction< - void Function(ggml_opt_dataset_t, ffi.Pointer, int, - ffi.Pointer, int)>(); + late final _gguf_get_n_tensorsPtr = _lookup< + ffi.NativeFunction)>>( + 'gguf_get_n_tensors'); + late final _gguf_get_n_tensors = _gguf_get_n_tensorsPtr + .asFunction)>(); - ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params( - ffi.Pointer userdata, + ffi.Pointer gguf_get_tensor_name( + ffi.Pointer ctx, + int tensor_id, ) { - return _ggml_opt_get_default_optimizer_params( - userdata, + return _gguf_get_tensor_name( + ctx, + tensor_id, ); } - late final _ggml_opt_get_default_optimizer_paramsPtr = _lookup< + late final _gguf_get_tensor_namePtr = _lookup< ffi.NativeFunction< - ggml_opt_optimizer_params Function( - ffi.Pointer)>>('ggml_opt_get_default_optimizer_params'); - late final _ggml_opt_get_default_optimizer_params = - _ggml_opt_get_default_optimizer_paramsPtr.asFunction< - ggml_opt_optimizer_params Function(ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_name'); + late final _gguf_get_tensor_name = _gguf_get_tensor_namePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params( - ffi.Pointer userdata, + ffi.Pointer gguf_get_tensor_ne( + ffi.Pointer ctx, + int tensor_id, ) { - return _ggml_opt_get_constant_optimizer_params( - userdata, + return _gguf_get_tensor_ne( + ctx, + tensor_id, ); } - late final _ggml_opt_get_constant_optimizer_paramsPtr = _lookup< - ffi.NativeFunction< - ggml_opt_optimizer_params Function(ffi.Pointer)>>( - 'ggml_opt_get_constant_optimizer_params'); - late final _ggml_opt_get_constant_optimizer_params = - _ggml_opt_get_constant_optimizer_paramsPtr.asFunction< - ggml_opt_optimizer_params Function(ffi.Pointer)>(); + late final _gguf_get_tensor_nePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_ne'); + late final _gguf_get_tensor_ne = _gguf_get_tensor_nePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - ggml_opt_params ggml_opt_default_params( - ggml_backend_sched_t backend_sched, - ggml_opt_loss_type loss_type, + int gguf_get_tensor_offset( + ffi.Pointer ctx, + int tensor_id, ) { - return _ggml_opt_default_params( - backend_sched, - loss_type.value, + return _gguf_get_tensor_offset( + ctx, + tensor_id, ); } - late final _ggml_opt_default_paramsPtr = _lookup< + late final _gguf_get_tensor_offsetPtr = _lookup< ffi.NativeFunction< - ggml_opt_params Function(ggml_backend_sched_t, - ffi.UnsignedInt)>>('ggml_opt_default_params'); - late final _ggml_opt_default_params = _ggml_opt_default_paramsPtr - .asFunction(); + ffi.Size Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_offset'); + late final _gguf_get_tensor_offset = _gguf_get_tensor_offsetPtr + .asFunction, int)>(); - ggml_opt_context_t ggml_opt_init( - ggml_opt_params params, + int gguf_get_tensor_size( + ffi.Pointer ctx, + int tensor_id, ) { - return _ggml_opt_init( - params, + return _gguf_get_tensor_size( + ctx, + tensor_id, ); } - late final _ggml_opt_initPtr = - _lookup>( - 'ggml_opt_init'); - late final _ggml_opt_init = _ggml_opt_initPtr - .asFunction(); + late final _gguf_get_tensor_sizePtr = _lookup< + ffi.NativeFunction< + ffi.Size Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_size'); + late final _gguf_get_tensor_size = _gguf_get_tensor_sizePtr + .asFunction, int)>(); - void ggml_opt_free( - ggml_opt_context_t opt_ctx, + ggml_type gguf_get_tensor_type( + ffi.Pointer ctx, + int tensor_id, ) { - return _ggml_opt_free( - opt_ctx, - ); + return ggml_type.fromValue(_gguf_get_tensor_type( + ctx, + tensor_id, + )); } - late final _ggml_opt_freePtr = - _lookup>( - 'ggml_opt_free'); - late final _ggml_opt_free = - _ggml_opt_freePtr.asFunction(); + late final _gguf_get_tensor_typePtr = _lookup< + ffi.NativeFunction< + ffi.UnsignedInt Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_type'); + late final _gguf_get_tensor_type = _gguf_get_tensor_typePtr + .asFunction, int)>(); - void ggml_opt_reset( - ggml_opt_context_t opt_ctx, - bool optimizer, + bool gguf_get_val_bool( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_reset( - opt_ctx, - optimizer, + return _gguf_get_val_bool( + ctx, + key_id, ); } - late final _ggml_opt_resetPtr = _lookup< - ffi.NativeFunction>( - 'ggml_opt_reset'); - late final _ggml_opt_reset = - _ggml_opt_resetPtr.asFunction(); - - bool ggml_opt_static_graphs( - ggml_opt_context_t opt_ctx, + late final _gguf_get_val_boolPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_bool'); + late final _gguf_get_val_bool = _gguf_get_val_boolPtr + .asFunction, int)>(); + + ffi.Pointer gguf_get_val_data( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_static_graphs( - opt_ctx, + return _gguf_get_val_data( + ctx, + key_id, ); } - late final _ggml_opt_static_graphsPtr = - _lookup>( - 'ggml_opt_static_graphs'); - late final _ggml_opt_static_graphs = _ggml_opt_static_graphsPtr - .asFunction(); + late final _gguf_get_val_dataPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_data'); + late final _gguf_get_val_data = _gguf_get_val_dataPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - ffi.Pointer ggml_opt_inputs( - ggml_opt_context_t opt_ctx, + double gguf_get_val_f32( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_inputs( - opt_ctx, + return _gguf_get_val_f32( + ctx, + key_id, ); } - late final _ggml_opt_inputsPtr = _lookup< + late final _gguf_get_val_f32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_opt_context_t)>>('ggml_opt_inputs'); - late final _ggml_opt_inputs = _ggml_opt_inputsPtr - .asFunction Function(ggml_opt_context_t)>(); + ffi.Float Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_f32'); + late final _gguf_get_val_f32 = _gguf_get_val_f32Ptr + .asFunction, int)>(); - ffi.Pointer ggml_opt_outputs( - ggml_opt_context_t opt_ctx, + double gguf_get_val_f64( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_outputs( - opt_ctx, + return _gguf_get_val_f64( + ctx, + key_id, ); } - late final _ggml_opt_outputsPtr = _lookup< + late final _gguf_get_val_f64Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_opt_context_t)>>('ggml_opt_outputs'); - late final _ggml_opt_outputs = _ggml_opt_outputsPtr - .asFunction Function(ggml_opt_context_t)>(); + ffi.Double Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_f64'); + late final _gguf_get_val_f64 = _gguf_get_val_f64Ptr + .asFunction, int)>(); - ffi.Pointer ggml_opt_labels( - ggml_opt_context_t opt_ctx, + int gguf_get_val_i16( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_labels( - opt_ctx, + return _gguf_get_val_i16( + ctx, + key_id, ); } - late final _ggml_opt_labelsPtr = _lookup< + late final _gguf_get_val_i16Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_opt_context_t)>>('ggml_opt_labels'); - late final _ggml_opt_labels = _ggml_opt_labelsPtr - .asFunction Function(ggml_opt_context_t)>(); + ffi.Int16 Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_i16'); + late final _gguf_get_val_i16 = _gguf_get_val_i16Ptr + .asFunction, int)>(); - ffi.Pointer ggml_opt_loss( - ggml_opt_context_t opt_ctx, + int gguf_get_val_i32( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_loss( - opt_ctx, + return _gguf_get_val_i32( + ctx, + key_id, ); } - late final _ggml_opt_lossPtr = _lookup< + late final _gguf_get_val_i32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_opt_context_t)>>('ggml_opt_loss'); - late final _ggml_opt_loss = _ggml_opt_lossPtr - .asFunction Function(ggml_opt_context_t)>(); + ffi.Int32 Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_i32'); + late final _gguf_get_val_i32 = _gguf_get_val_i32Ptr + .asFunction, int)>(); - ffi.Pointer ggml_opt_pred( - ggml_opt_context_t opt_ctx, + int gguf_get_val_i64( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_pred( - opt_ctx, + return _gguf_get_val_i64( + ctx, + key_id, ); } - late final _ggml_opt_predPtr = _lookup< + late final _gguf_get_val_i64Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_opt_context_t)>>('ggml_opt_pred'); - late final _ggml_opt_pred = _ggml_opt_predPtr - .asFunction Function(ggml_opt_context_t)>(); + ffi.Int64 Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_i64'); + late final _gguf_get_val_i64 = _gguf_get_val_i64Ptr + .asFunction, int)>(); - ffi.Pointer ggml_opt_ncorrect( - ggml_opt_context_t opt_ctx, + int gguf_get_val_i8( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_ncorrect( - opt_ctx, + return _gguf_get_val_i8( + ctx, + key_id, ); } - late final _ggml_opt_ncorrectPtr = _lookup< + late final _gguf_get_val_i8Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ggml_opt_context_t)>>('ggml_opt_ncorrect'); - late final _ggml_opt_ncorrect = _ggml_opt_ncorrectPtr - .asFunction Function(ggml_opt_context_t)>(); + ffi.Int8 Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_i8'); + late final _gguf_get_val_i8 = _gguf_get_val_i8Ptr + .asFunction, int)>(); - ffi.Pointer ggml_opt_grad_acc( - ggml_opt_context_t opt_ctx, - ffi.Pointer node, + ffi.Pointer gguf_get_val_str( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_grad_acc( - opt_ctx, - node, + return _gguf_get_val_str( + ctx, + key_id, ); } - late final _ggml_opt_grad_accPtr = _lookup< + late final _gguf_get_val_strPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ggml_opt_context_t, - ffi.Pointer)>>('ggml_opt_grad_acc'); - late final _ggml_opt_grad_acc = _ggml_opt_grad_accPtr.asFunction< - ffi.Pointer Function( - ggml_opt_context_t, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_str'); + late final _gguf_get_val_str = _gguf_get_val_strPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - ggml_opt_optimizer_type ggml_opt_context_optimizer_type( - ggml_opt_context_t arg0, + int gguf_get_val_u16( + ffi.Pointer ctx, + int key_id, ) { - return ggml_opt_optimizer_type.fromValue(_ggml_opt_context_optimizer_type( - arg0, - )); + return _gguf_get_val_u16( + ctx, + key_id, + ); } - late final _ggml_opt_context_optimizer_typePtr = - _lookup>( - 'ggml_opt_context_optimizer_type'); - late final _ggml_opt_context_optimizer_type = - _ggml_opt_context_optimizer_typePtr - .asFunction(); + late final _gguf_get_val_u16Ptr = _lookup< + ffi.NativeFunction< + ffi.Uint16 Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_u16'); + late final _gguf_get_val_u16 = _gguf_get_val_u16Ptr + .asFunction, int)>(); - ffi.Pointer ggml_opt_optimizer_name( - ggml_opt_optimizer_type arg0, + int gguf_get_val_u32( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_optimizer_name( - arg0.value, + return _gguf_get_val_u32( + ctx, + key_id, ); } - late final _ggml_opt_optimizer_namePtr = _lookup< - ffi.NativeFunction Function(ffi.UnsignedInt)>>( - 'ggml_opt_optimizer_name'); - late final _ggml_opt_optimizer_name = _ggml_opt_optimizer_namePtr - .asFunction Function(int)>(); - - ggml_opt_result_t ggml_opt_result_init() { - return _ggml_opt_result_init(); - } - - late final _ggml_opt_result_initPtr = - _lookup>( - 'ggml_opt_result_init'); - late final _ggml_opt_result_init = - _ggml_opt_result_initPtr.asFunction(); + late final _gguf_get_val_u32Ptr = _lookup< + ffi.NativeFunction< + ffi.Uint32 Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_u32'); + late final _gguf_get_val_u32 = _gguf_get_val_u32Ptr + .asFunction, int)>(); - void ggml_opt_result_free( - ggml_opt_result_t result, + int gguf_get_val_u64( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_result_free( - result, + return _gguf_get_val_u64( + ctx, + key_id, ); } - late final _ggml_opt_result_freePtr = - _lookup>( - 'ggml_opt_result_free'); - late final _ggml_opt_result_free = - _ggml_opt_result_freePtr.asFunction(); + late final _gguf_get_val_u64Ptr = _lookup< + ffi.NativeFunction< + ffi.Uint64 Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_u64'); + late final _gguf_get_val_u64 = _gguf_get_val_u64Ptr + .asFunction, int)>(); - void ggml_opt_result_reset( - ggml_opt_result_t result, + int gguf_get_val_u8( + ffi.Pointer ctx, + int key_id, ) { - return _ggml_opt_result_reset( - result, + return _gguf_get_val_u8( + ctx, + key_id, ); } - late final _ggml_opt_result_resetPtr = - _lookup>( - 'ggml_opt_result_reset'); - late final _ggml_opt_result_reset = - _ggml_opt_result_resetPtr.asFunction(); + late final _gguf_get_val_u8Ptr = _lookup< + ffi.NativeFunction< + ffi.Uint8 Function( + ffi.Pointer, ffi.Int64)>>('gguf_get_val_u8'); + late final _gguf_get_val_u8 = _gguf_get_val_u8Ptr + .asFunction, int)>(); - void ggml_opt_result_ndata( - ggml_opt_result_t result, - ffi.Pointer ndata, + int gguf_get_version( + ffi.Pointer ctx, ) { - return _ggml_opt_result_ndata( - result, - ndata, + return _gguf_get_version( + ctx, ); } - late final _ggml_opt_result_ndataPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ggml_opt_result_t, - ffi.Pointer)>>('ggml_opt_result_ndata'); - late final _ggml_opt_result_ndata = _ggml_opt_result_ndataPtr - .asFunction)>(); + late final _gguf_get_versionPtr = _lookup< + ffi.NativeFunction)>>( + 'gguf_get_version'); + late final _gguf_get_version = _gguf_get_versionPtr + .asFunction)>(); - void ggml_opt_result_loss( - ggml_opt_result_t result, - ffi.Pointer loss, - ffi.Pointer unc, + ffi.Pointer gguf_init_empty() { + return _gguf_init_empty(); + } + + late final _gguf_init_emptyPtr = + _lookup Function()>>( + 'gguf_init_empty'); + late final _gguf_init_empty = + _gguf_init_emptyPtr.asFunction Function()>(); + + ffi.Pointer gguf_init_from_buffer( + ffi.Pointer data, + int size, + gguf_init_params params, ) { - return _ggml_opt_result_loss( - result, - loss, - unc, + return _gguf_init_from_buffer( + data, + size, + params, ); } - late final _ggml_opt_result_lossPtr = _lookup< + late final _gguf_init_from_bufferPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_opt_result_t, ffi.Pointer, - ffi.Pointer)>>('ggml_opt_result_loss'); - late final _ggml_opt_result_loss = _ggml_opt_result_lossPtr.asFunction< - void Function(ggml_opt_result_t, ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, ffi.Size, + gguf_init_params)>>('gguf_init_from_buffer'); + late final _gguf_init_from_buffer = _gguf_init_from_bufferPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, int, gguf_init_params)>(); - void ggml_opt_result_pred( - ggml_opt_result_t result, - ffi.Pointer pred, + ffi.Pointer gguf_init_from_callback( + gguf_reader_callback_t callback, + ffi.Pointer userdata, + int max_chunk_read, + int max_expected_size, + gguf_init_params params, ) { - return _ggml_opt_result_pred( - result, - pred, + return _gguf_init_from_callback( + callback, + userdata, + max_chunk_read, + max_expected_size, + params, ); } - late final _ggml_opt_result_predPtr = _lookup< + late final _gguf_init_from_callbackPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_opt_result_t, - ffi.Pointer)>>('ggml_opt_result_pred'); - late final _ggml_opt_result_pred = _ggml_opt_result_predPtr - .asFunction)>(); + ffi.Pointer Function( + gguf_reader_callback_t, + ffi.Pointer, + ffi.Size, + ffi.Uint64, + gguf_init_params)>>('gguf_init_from_callback'); + late final _gguf_init_from_callback = _gguf_init_from_callbackPtr.asFunction< + ffi.Pointer Function(gguf_reader_callback_t, + ffi.Pointer, int, int, gguf_init_params)>(); - void ggml_opt_result_accuracy( - ggml_opt_result_t result, - ffi.Pointer accuracy, - ffi.Pointer unc, + ffi.Pointer gguf_init_from_file( + ffi.Pointer fname, + gguf_init_params params, ) { - return _ggml_opt_result_accuracy( - result, - accuracy, - unc, + return _gguf_init_from_file( + fname, + params, ); } - late final _ggml_opt_result_accuracyPtr = _lookup< + late final _gguf_init_from_filePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ggml_opt_result_t, ffi.Pointer, - ffi.Pointer)>>('ggml_opt_result_accuracy'); - late final _ggml_opt_result_accuracy = - _ggml_opt_result_accuracyPtr.asFunction< - void Function(ggml_opt_result_t, ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, gguf_init_params)>>('gguf_init_from_file'); + late final _gguf_init_from_file = _gguf_init_from_filePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, gguf_init_params)>(); - void ggml_opt_prepare_alloc( - ggml_opt_context_t opt_ctx, - ffi.Pointer ctx_compute, - ffi.Pointer gf, - ffi.Pointer inputs, - ffi.Pointer outputs, + ffi.Pointer gguf_init_from_file_ptr( + ffi.Pointer file, + gguf_init_params params, ) { - return _ggml_opt_prepare_alloc( - opt_ctx, - ctx_compute, - gf, - inputs, - outputs, + return _gguf_init_from_file_ptr( + file, + params, ); } - late final _ggml_opt_prepare_allocPtr = _lookup< + late final _gguf_init_from_file_ptrPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_opt_context_t, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('ggml_opt_prepare_alloc'); - late final _ggml_opt_prepare_alloc = _ggml_opt_prepare_allocPtr.asFunction< - void Function( - ggml_opt_context_t, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, gguf_init_params)>>('gguf_init_from_file_ptr'); + late final _gguf_init_from_file_ptr = _gguf_init_from_file_ptrPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, gguf_init_params)>(); - void ggml_opt_alloc( - ggml_opt_context_t opt_ctx, - bool backward, + int gguf_remove_key( + ffi.Pointer ctx, + ffi.Pointer key, ) { - return _ggml_opt_alloc( - opt_ctx, - backward, + return _gguf_remove_key( + ctx, + key, ); } - late final _ggml_opt_allocPtr = _lookup< - ffi.NativeFunction>( - 'ggml_opt_alloc'); - late final _ggml_opt_alloc = - _ggml_opt_allocPtr.asFunction(); + late final _gguf_remove_keyPtr = _lookup< + ffi.NativeFunction< + ffi.Int64 Function(ffi.Pointer, + ffi.Pointer)>>('gguf_remove_key'); + late final _gguf_remove_key = _gguf_remove_keyPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - void ggml_opt_eval( - ggml_opt_context_t opt_ctx, - ggml_opt_result_t result, + void gguf_set_arr_data( + ffi.Pointer ctx, + ffi.Pointer key, + gguf_type type, + ffi.Pointer data, + int n, ) { - return _ggml_opt_eval( - opt_ctx, - result, + return _gguf_set_arr_data( + ctx, + key, + type.value, + data, + n, ); } - late final _ggml_opt_evalPtr = _lookup< + late final _gguf_set_arr_dataPtr = _lookup< ffi.NativeFunction< ffi.Void Function( - ggml_opt_context_t, ggml_opt_result_t)>>('ggml_opt_eval'); - late final _ggml_opt_eval = _ggml_opt_evalPtr - .asFunction(); + ffi.Pointer, + ffi.Pointer, + ffi.UnsignedInt, + ffi.Pointer, + ffi.Size)>>('gguf_set_arr_data'); + late final _gguf_set_arr_data = _gguf_set_arr_dataPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int, + ffi.Pointer, int)>(); - void ggml_opt_epoch( - ggml_opt_context_t opt_ctx, - ggml_opt_dataset_t dataset, - ggml_opt_result_t result_train, - ggml_opt_result_t result_eval, - int idata_split, - ggml_opt_epoch_callback callback_train, - ggml_opt_epoch_callback callback_eval, + void gguf_set_arr_str( + ffi.Pointer ctx, + ffi.Pointer key, + ffi.Pointer> data, + int n, ) { - return _ggml_opt_epoch( - opt_ctx, - dataset, - result_train, - result_eval, - idata_split, - callback_train, - callback_eval, + return _gguf_set_arr_str( + ctx, + key, + data, + n, ); } - late final _ggml_opt_epochPtr = _lookup< + late final _gguf_set_arr_strPtr = _lookup< ffi.NativeFunction< ffi.Void Function( - ggml_opt_context_t, - ggml_opt_dataset_t, - ggml_opt_result_t, - ggml_opt_result_t, - ffi.Int64, - ggml_opt_epoch_callback, - ggml_opt_epoch_callback)>>('ggml_opt_epoch'); - late final _ggml_opt_epoch = _ggml_opt_epochPtr.asFunction< - void Function( - ggml_opt_context_t, - ggml_opt_dataset_t, - ggml_opt_result_t, - ggml_opt_result_t, - int, - ggml_opt_epoch_callback, - ggml_opt_epoch_callback)>(); + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>, + ffi.Size)>>('gguf_set_arr_str'); + late final _gguf_set_arr_str = _gguf_set_arr_strPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer>, int)>(); - void ggml_opt_epoch_callback_progress_bar( - bool train, - ggml_opt_context_t opt_ctx, - ggml_opt_dataset_t dataset, - ggml_opt_result_t result, - int ibatch, - int ibatch_max, - int t_start_us, + void gguf_set_kv( + ffi.Pointer ctx, + ffi.Pointer src, ) { - return _ggml_opt_epoch_callback_progress_bar( - train, - opt_ctx, - dataset, - result, - ibatch, - ibatch_max, - t_start_us, + return _gguf_set_kv( + ctx, + src, ); } - late final _ggml_opt_epoch_callback_progress_barPtr = _lookup< + late final _gguf_set_kvPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ffi.Bool, - ggml_opt_context_t, - ggml_opt_dataset_t, - ggml_opt_result_t, - ffi.Int64, - ffi.Int64, - ffi.Int64)>>('ggml_opt_epoch_callback_progress_bar'); - late final _ggml_opt_epoch_callback_progress_bar = - _ggml_opt_epoch_callback_progress_barPtr.asFunction< - void Function(bool, ggml_opt_context_t, ggml_opt_dataset_t, - ggml_opt_result_t, int, int, int)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('gguf_set_kv'); + late final _gguf_set_kv = _gguf_set_kvPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - void ggml_opt_fit( - ggml_backend_sched_t backend_sched, - ffi.Pointer ctx_compute, - ffi.Pointer inputs, - ffi.Pointer outputs, - ggml_opt_dataset_t dataset, - ggml_opt_loss_type loss_type, - ggml_opt_optimizer_type optimizer, - ggml_opt_get_optimizer_params get_opt_pars, - int nepoch, - int nbatch_logical, - double val_split, - bool silent, + void gguf_set_tensor_data( + ffi.Pointer ctx, + ffi.Pointer name, + ffi.Pointer data, ) { - return _ggml_opt_fit( - backend_sched, - ctx_compute, - inputs, - outputs, - dataset, - loss_type.value, - optimizer.value, - get_opt_pars, - nepoch, - nbatch_logical, - val_split, - silent, + return _gguf_set_tensor_data( + ctx, + name, + data, ); } - late final _ggml_opt_fitPtr = _lookup< + late final _gguf_set_tensor_dataPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_backend_sched_t, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_opt_dataset_t, - ffi.UnsignedInt, - ffi.UnsignedInt, - ggml_opt_get_optimizer_params, - ffi.Int64, - ffi.Int64, - ffi.Float, - ffi.Bool)>>('ggml_opt_fit'); - late final _ggml_opt_fit = _ggml_opt_fitPtr.asFunction< - void Function( - ggml_backend_sched_t, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ggml_opt_dataset_t, - int, - int, - ggml_opt_get_optimizer_params, - int, - int, - double, - bool)>(); - - ffi.Pointer gguf_init_empty() { - return _gguf_init_empty(); - } - - late final _gguf_init_emptyPtr = - _lookup Function()>>( - 'gguf_init_empty'); - late final _gguf_init_empty = - _gguf_init_emptyPtr.asFunction Function()>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer)>>('gguf_set_tensor_data'); + late final _gguf_set_tensor_data = _gguf_set_tensor_dataPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer)>(); - ffi.Pointer gguf_init_from_file_ptr( - ffi.Pointer file, - gguf_init_params params, + void gguf_set_tensor_type( + ffi.Pointer ctx, + ffi.Pointer name, + ggml_type type, ) { - return _gguf_init_from_file_ptr( - file, - params, + return _gguf_set_tensor_type( + ctx, + name, + type.value, ); } - late final _gguf_init_from_file_ptrPtr = _lookup< + late final _gguf_set_tensor_typePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, gguf_init_params)>>('gguf_init_from_file_ptr'); - late final _gguf_init_from_file_ptr = _gguf_init_from_file_ptrPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, gguf_init_params)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.UnsignedInt)>>('gguf_set_tensor_type'); + late final _gguf_set_tensor_type = _gguf_set_tensor_typePtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer gguf_init_from_file( - ffi.Pointer fname, - gguf_init_params params, + void gguf_set_val_bool( + ffi.Pointer ctx, + ffi.Pointer key, + bool val, ) { - return _gguf_init_from_file( - fname, - params, + return _gguf_set_val_bool( + ctx, + key, + val, ); } - late final _gguf_init_from_filePtr = _lookup< + late final _gguf_set_val_boolPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, gguf_init_params)>>('gguf_init_from_file'); - late final _gguf_init_from_file = _gguf_init_from_filePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, gguf_init_params)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Bool)>>('gguf_set_val_bool'); + late final _gguf_set_val_bool = _gguf_set_val_boolPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, bool)>(); - ffi.Pointer gguf_init_from_buffer( - ffi.Pointer data, - int size, - gguf_init_params params, + void gguf_set_val_f32( + ffi.Pointer ctx, + ffi.Pointer key, + double val, ) { - return _gguf_init_from_buffer( - data, - size, - params, + return _gguf_set_val_f32( + ctx, + key, + val, ); } - late final _gguf_init_from_bufferPtr = _lookup< + late final _gguf_set_val_f32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, ffi.Size, - gguf_init_params)>>('gguf_init_from_buffer'); - late final _gguf_init_from_buffer = _gguf_init_from_bufferPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int, gguf_init_params)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Float)>>('gguf_set_val_f32'); + late final _gguf_set_val_f32 = _gguf_set_val_f32Ptr.asFunction< + void Function( + ffi.Pointer, ffi.Pointer, double)>(); - ffi.Pointer gguf_init_from_callback( - gguf_reader_callback_t callback, - ffi.Pointer userdata, - int max_chunk_read, - int max_expected_size, - gguf_init_params params, + void gguf_set_val_f64( + ffi.Pointer ctx, + ffi.Pointer key, + double val, ) { - return _gguf_init_from_callback( - callback, - userdata, - max_chunk_read, - max_expected_size, - params, + return _gguf_set_val_f64( + ctx, + key, + val, ); } - late final _gguf_init_from_callbackPtr = _lookup< + late final _gguf_set_val_f64Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - gguf_reader_callback_t, - ffi.Pointer, - ffi.Size, - ffi.Uint64, - gguf_init_params)>>('gguf_init_from_callback'); - late final _gguf_init_from_callback = _gguf_init_from_callbackPtr.asFunction< - ffi.Pointer Function(gguf_reader_callback_t, - ffi.Pointer, int, int, gguf_init_params)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Double)>>('gguf_set_val_f64'); + late final _gguf_set_val_f64 = _gguf_set_val_f64Ptr.asFunction< + void Function( + ffi.Pointer, ffi.Pointer, double)>(); - void gguf_free( + void gguf_set_val_i16( ffi.Pointer ctx, + ffi.Pointer key, + int val, ) { - return _gguf_free( + return _gguf_set_val_i16( ctx, + key, + val, ); } - late final _gguf_freePtr = - _lookup)>>( - 'gguf_free'); - late final _gguf_free = - _gguf_freePtr.asFunction)>(); - - ffi.Pointer gguf_type_name( - gguf_type type, - ) { - return _gguf_type_name( - type.value, - ); - } - - late final _gguf_type_namePtr = _lookup< - ffi.NativeFunction Function(ffi.UnsignedInt)>>( - 'gguf_type_name'); - late final _gguf_type_name = - _gguf_type_namePtr.asFunction Function(int)>(); + late final _gguf_set_val_i16Ptr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int16)>>('gguf_set_val_i16'); + late final _gguf_set_val_i16 = _gguf_set_val_i16Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - int gguf_get_version( + void gguf_set_val_i32( ffi.Pointer ctx, + ffi.Pointer key, + int val, ) { - return _gguf_get_version( + return _gguf_set_val_i32( ctx, + key, + val, ); } - late final _gguf_get_versionPtr = _lookup< - ffi.NativeFunction)>>( - 'gguf_get_version'); - late final _gguf_get_version = _gguf_get_versionPtr - .asFunction)>(); + late final _gguf_set_val_i32Ptr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int32)>>('gguf_set_val_i32'); + late final _gguf_set_val_i32 = _gguf_set_val_i32Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - int gguf_get_alignment( + void gguf_set_val_i64( ffi.Pointer ctx, + ffi.Pointer key, + int val, ) { - return _gguf_get_alignment( + return _gguf_set_val_i64( ctx, + key, + val, ); } - late final _gguf_get_alignmentPtr = - _lookup)>>( - 'gguf_get_alignment'); - late final _gguf_get_alignment = _gguf_get_alignmentPtr - .asFunction)>(); + late final _gguf_set_val_i64Ptr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int64)>>('gguf_set_val_i64'); + late final _gguf_set_val_i64 = _gguf_set_val_i64Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - int gguf_get_data_offset( + void gguf_set_val_i8( ffi.Pointer ctx, + ffi.Pointer key, + int val, ) { - return _gguf_get_data_offset( + return _gguf_set_val_i8( ctx, + key, + val, ); } - late final _gguf_get_data_offsetPtr = - _lookup)>>( - 'gguf_get_data_offset'); - late final _gguf_get_data_offset = _gguf_get_data_offsetPtr - .asFunction)>(); + late final _gguf_set_val_i8Ptr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Int8)>>('gguf_set_val_i8'); + late final _gguf_set_val_i8 = _gguf_set_val_i8Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - int gguf_get_n_kv( + void gguf_set_val_str( ffi.Pointer ctx, + ffi.Pointer key, + ffi.Pointer val, ) { - return _gguf_get_n_kv( + return _gguf_set_val_str( ctx, + key, + val, ); } - late final _gguf_get_n_kvPtr = _lookup< - ffi.NativeFunction)>>( - 'gguf_get_n_kv'); - late final _gguf_get_n_kv = - _gguf_get_n_kvPtr.asFunction)>(); + late final _gguf_set_val_strPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer)>>('gguf_set_val_str'); + late final _gguf_set_val_str = _gguf_set_val_strPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer)>(); - int gguf_find_key( + void gguf_set_val_u16( ffi.Pointer ctx, ffi.Pointer key, + int val, ) { - return _gguf_find_key( + return _gguf_set_val_u16( ctx, key, + val, ); } - late final _gguf_find_keyPtr = _lookup< + late final _gguf_set_val_u16Ptr = _lookup< ffi.NativeFunction< - ffi.Int64 Function(ffi.Pointer, - ffi.Pointer)>>('gguf_find_key'); - late final _gguf_find_key = _gguf_find_keyPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Uint16)>>('gguf_set_val_u16'); + late final _gguf_set_val_u16 = _gguf_set_val_u16Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer gguf_get_key( + void gguf_set_val_u32( ffi.Pointer ctx, - int key_id, + ffi.Pointer key, + int val, ) { - return _gguf_get_key( + return _gguf_set_val_u32( ctx, - key_id, + key, + val, ); } - late final _gguf_get_keyPtr = _lookup< + late final _gguf_set_val_u32Ptr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_key'); - late final _gguf_get_key = _gguf_get_keyPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Uint32)>>('gguf_set_val_u32'); + late final _gguf_set_val_u32 = _gguf_set_val_u32Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - gguf_type gguf_get_kv_type( + void gguf_set_val_u64( ffi.Pointer ctx, - int key_id, + ffi.Pointer key, + int val, ) { - return gguf_type.fromValue(_gguf_get_kv_type( + return _gguf_set_val_u64( ctx, - key_id, - )); + key, + val, + ); } - late final _gguf_get_kv_typePtr = _lookup< + late final _gguf_set_val_u64Ptr = _lookup< ffi.NativeFunction< - ffi.UnsignedInt Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_kv_type'); - late final _gguf_get_kv_type = _gguf_get_kv_typePtr - .asFunction, int)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Uint64)>>('gguf_set_val_u64'); + late final _gguf_set_val_u64 = _gguf_set_val_u64Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - gguf_type gguf_get_arr_type( + void gguf_set_val_u8( ffi.Pointer ctx, - int key_id, + ffi.Pointer key, + int val, ) { - return gguf_type.fromValue(_gguf_get_arr_type( + return _gguf_set_val_u8( ctx, - key_id, - )); + key, + val, + ); } - late final _gguf_get_arr_typePtr = _lookup< + late final _gguf_set_val_u8Ptr = _lookup< ffi.NativeFunction< - ffi.UnsignedInt Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_arr_type'); - late final _gguf_get_arr_type = _gguf_get_arr_typePtr - .asFunction, int)>(); + ffi.Void Function(ffi.Pointer, ffi.Pointer, + ffi.Uint8)>>('gguf_set_val_u8'); + late final _gguf_set_val_u8 = _gguf_set_val_u8Ptr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); - int gguf_get_val_u8( - ffi.Pointer ctx, - int key_id, + ffi.Pointer gguf_type_name( + gguf_type type, ) { - return _gguf_get_val_u8( - ctx, - key_id, + return _gguf_type_name( + type.value, ); } - late final _gguf_get_val_u8Ptr = _lookup< - ffi.NativeFunction< - ffi.Uint8 Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_u8'); - late final _gguf_get_val_u8 = _gguf_get_val_u8Ptr - .asFunction, int)>(); + late final _gguf_type_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'gguf_type_name'); + late final _gguf_type_name = + _gguf_type_namePtr.asFunction Function(int)>(); - int gguf_get_val_i8( + bool gguf_write_to_file( ffi.Pointer ctx, - int key_id, + ffi.Pointer fname, + bool only_meta, ) { - return _gguf_get_val_i8( + return _gguf_write_to_file( ctx, - key_id, + fname, + only_meta, ); } - late final _gguf_get_val_i8Ptr = _lookup< + late final _gguf_write_to_filePtr = _lookup< ffi.NativeFunction< - ffi.Int8 Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_i8'); - late final _gguf_get_val_i8 = _gguf_get_val_i8Ptr - .asFunction, int)>(); + ffi.Bool Function(ffi.Pointer, ffi.Pointer, + ffi.Bool)>>('gguf_write_to_file'); + late final _gguf_write_to_file = _gguf_write_to_filePtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer, bool)>(); - int gguf_get_val_u16( + bool gguf_write_to_file_ptr( ffi.Pointer ctx, - int key_id, + ffi.Pointer file, + bool only_meta, ) { - return _gguf_get_val_u16( + return _gguf_write_to_file_ptr( ctx, - key_id, + file, + only_meta, ); } - late final _gguf_get_val_u16Ptr = _lookup< + late final _gguf_write_to_file_ptrPtr = _lookup< ffi.NativeFunction< - ffi.Uint16 Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_u16'); - late final _gguf_get_val_u16 = _gguf_get_val_u16Ptr - .asFunction, int)>(); + ffi.Bool Function(ffi.Pointer, ffi.Pointer, + ffi.Bool)>>('gguf_write_to_file_ptr'); + late final _gguf_write_to_file_ptr = _gguf_write_to_file_ptrPtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer, bool)>(); - int gguf_get_val_i16( - ffi.Pointer ctx, - int key_id, + ffi.Pointer llama_adapter_get_alora_invocation_tokens( + ffi.Pointer adapter, ) { - return _gguf_get_val_i16( - ctx, - key_id, + return _llama_adapter_get_alora_invocation_tokens( + adapter, ); } - late final _gguf_get_val_i16Ptr = _lookup< - ffi.NativeFunction< - ffi.Int16 Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_i16'); - late final _gguf_get_val_i16 = _gguf_get_val_i16Ptr - .asFunction, int)>(); + late final _llama_adapter_get_alora_invocation_tokensPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer)>>( + 'llama_adapter_get_alora_invocation_tokens'); + late final _llama_adapter_get_alora_invocation_tokens = + _llama_adapter_get_alora_invocation_tokensPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - int gguf_get_val_u32( - ffi.Pointer ctx, - int key_id, + int llama_adapter_get_alora_n_invocation_tokens( + ffi.Pointer adapter, ) { - return _gguf_get_val_u32( - ctx, - key_id, + return _llama_adapter_get_alora_n_invocation_tokens( + adapter, ); } - late final _gguf_get_val_u32Ptr = _lookup< - ffi.NativeFunction< - ffi.Uint32 Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_u32'); - late final _gguf_get_val_u32 = _gguf_get_val_u32Ptr - .asFunction, int)>(); + late final _llama_adapter_get_alora_n_invocation_tokensPtr = _lookup< + ffi.NativeFunction< + ffi.Uint64 Function(ffi.Pointer)>>( + 'llama_adapter_get_alora_n_invocation_tokens'); + late final _llama_adapter_get_alora_n_invocation_tokens = + _llama_adapter_get_alora_n_invocation_tokensPtr + .asFunction)>(); - int gguf_get_val_i32( - ffi.Pointer ctx, - int key_id, + void llama_adapter_lora_free( + ffi.Pointer adapter, ) { - return _gguf_get_val_i32( - ctx, - key_id, + return _llama_adapter_lora_free( + adapter, ); } - late final _gguf_get_val_i32Ptr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_i32'); - late final _gguf_get_val_i32 = _gguf_get_val_i32Ptr - .asFunction, int)>(); + late final _llama_adapter_lora_freePtr = _lookup< + ffi + .NativeFunction)>>( + 'llama_adapter_lora_free'); + late final _llama_adapter_lora_free = _llama_adapter_lora_freePtr + .asFunction)>(); - double gguf_get_val_f32( - ffi.Pointer ctx, - int key_id, + ffi.Pointer llama_adapter_lora_init( + ffi.Pointer model, + ffi.Pointer path_lora, ) { - return _gguf_get_val_f32( - ctx, - key_id, + return _llama_adapter_lora_init( + model, + path_lora, ); } - late final _gguf_get_val_f32Ptr = _lookup< + late final _llama_adapter_lora_initPtr = _lookup< ffi.NativeFunction< - ffi.Float Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_f32'); - late final _gguf_get_val_f32 = _gguf_get_val_f32Ptr - .asFunction, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('llama_adapter_lora_init'); + late final _llama_adapter_lora_init = _llama_adapter_lora_initPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - int gguf_get_val_u64( - ffi.Pointer ctx, - int key_id, + int llama_adapter_meta_count( + ffi.Pointer adapter, ) { - return _gguf_get_val_u64( - ctx, - key_id, + return _llama_adapter_meta_count( + adapter, ); } - late final _gguf_get_val_u64Ptr = _lookup< - ffi.NativeFunction< - ffi.Uint64 Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_u64'); - late final _gguf_get_val_u64 = _gguf_get_val_u64Ptr - .asFunction, int)>(); + late final _llama_adapter_meta_countPtr = _lookup< + ffi + .NativeFunction)>>( + 'llama_adapter_meta_count'); + late final _llama_adapter_meta_count = _llama_adapter_meta_countPtr + .asFunction)>(); - int gguf_get_val_i64( - ffi.Pointer ctx, - int key_id, + int llama_adapter_meta_key_by_index( + ffi.Pointer adapter, + int i, + ffi.Pointer buf, + int buf_size, ) { - return _gguf_get_val_i64( - ctx, - key_id, + return _llama_adapter_meta_key_by_index( + adapter, + i, + buf, + buf_size, ); } - late final _gguf_get_val_i64Ptr = _lookup< + late final _llama_adapter_meta_key_by_indexPtr = _lookup< ffi.NativeFunction< - ffi.Int64 Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_i64'); - late final _gguf_get_val_i64 = _gguf_get_val_i64Ptr - .asFunction, int)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Int32, + ffi.Pointer, + ffi.Size)>>('llama_adapter_meta_key_by_index'); + late final _llama_adapter_meta_key_by_index = + _llama_adapter_meta_key_by_indexPtr.asFunction< + int Function(ffi.Pointer, int, + ffi.Pointer, int)>(); - double gguf_get_val_f64( - ffi.Pointer ctx, - int key_id, + int llama_adapter_meta_val_str( + ffi.Pointer adapter, + ffi.Pointer key, + ffi.Pointer buf, + int buf_size, ) { - return _gguf_get_val_f64( - ctx, - key_id, + return _llama_adapter_meta_val_str( + adapter, + key, + buf, + buf_size, ); } - late final _gguf_get_val_f64Ptr = _lookup< + late final _llama_adapter_meta_val_strPtr = _lookup< ffi.NativeFunction< - ffi.Double Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_f64'); - late final _gguf_get_val_f64 = _gguf_get_val_f64Ptr - .asFunction, int)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size)>>('llama_adapter_meta_val_str'); + late final _llama_adapter_meta_val_str = + _llama_adapter_meta_val_strPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, int)>(); - bool gguf_get_val_bool( - ffi.Pointer ctx, - int key_id, + int llama_adapter_meta_val_str_by_index( + ffi.Pointer adapter, + int i, + ffi.Pointer buf, + int buf_size, ) { - return _gguf_get_val_bool( - ctx, - key_id, + return _llama_adapter_meta_val_str_by_index( + adapter, + i, + buf, + buf_size, ); } - late final _gguf_get_val_boolPtr = _lookup< + late final _llama_adapter_meta_val_str_by_indexPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_bool'); - late final _gguf_get_val_bool = _gguf_get_val_boolPtr - .asFunction, int)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Int32, + ffi.Pointer, + ffi.Size)>>('llama_adapter_meta_val_str_by_index'); + late final _llama_adapter_meta_val_str_by_index = + _llama_adapter_meta_val_str_by_indexPtr.asFunction< + int Function(ffi.Pointer, int, + ffi.Pointer, int)>(); - ffi.Pointer gguf_get_val_str( - ffi.Pointer ctx, - int key_id, + @Deprecated('use llama_vocab_get_add_bos instead') + bool llama_add_bos_token( + ffi.Pointer vocab, ) { - return _gguf_get_val_str( - ctx, - key_id, + return _llama_add_bos_token( + vocab, ); } - late final _gguf_get_val_strPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_str'); - late final _gguf_get_val_str = _gguf_get_val_strPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + late final _llama_add_bos_tokenPtr = + _lookup)>>( + 'llama_add_bos_token'); + late final _llama_add_bos_token = _llama_add_bos_tokenPtr + .asFunction)>(); - ffi.Pointer gguf_get_val_data( - ffi.Pointer ctx, - int key_id, + @Deprecated('use llama_vocab_get_add_eos instead') + bool llama_add_eos_token( + ffi.Pointer vocab, ) { - return _gguf_get_val_data( - ctx, - key_id, + return _llama_add_eos_token( + vocab, ); } - late final _gguf_get_val_dataPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_val_data'); - late final _gguf_get_val_data = _gguf_get_val_dataPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + late final _llama_add_eos_tokenPtr = + _lookup)>>( + 'llama_add_eos_token'); + late final _llama_add_eos_token = _llama_add_eos_tokenPtr + .asFunction)>(); - int gguf_get_arr_n( - ffi.Pointer ctx, - int key_id, + void llama_attach_threadpool( + ffi.Pointer ctx, + ggml_threadpool_t threadpool, + ggml_threadpool_t threadpool_batch, ) { - return _gguf_get_arr_n( + return _llama_attach_threadpool( ctx, - key_id, + threadpool, + threadpool_batch, ); } - late final _gguf_get_arr_nPtr = _lookup< + late final _llama_attach_threadpoolPtr = _lookup< ffi.NativeFunction< - ffi.Size Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_arr_n'); - late final _gguf_get_arr_n = _gguf_get_arr_nPtr - .asFunction, int)>(); - - ffi.Pointer gguf_get_arr_data( - ffi.Pointer ctx, - int key_id, - ) { - return _gguf_get_arr_data( - ctx, - key_id, - ); + ffi.Void Function(ffi.Pointer, ggml_threadpool_t, + ggml_threadpool_t)>>('llama_attach_threadpool'); + late final _llama_attach_threadpool = _llama_attach_threadpoolPtr.asFunction< + void Function( + ffi.Pointer, ggml_threadpool_t, ggml_threadpool_t)>(); + + void llama_backend_free() { + return _llama_backend_free(); } - late final _gguf_get_arr_dataPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_arr_data'); - late final _gguf_get_arr_data = _gguf_get_arr_dataPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + late final _llama_backend_freePtr = + _lookup>('llama_backend_free'); + late final _llama_backend_free = + _llama_backend_freePtr.asFunction(); - ffi.Pointer gguf_get_arr_str( - ffi.Pointer ctx, - int key_id, - int i, - ) { - return _gguf_get_arr_str( - ctx, - key_id, - i, - ); + void llama_backend_init() { + return _llama_backend_init(); } - late final _gguf_get_arr_strPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, ffi.Int64, - ffi.Size)>>('gguf_get_arr_str'); - late final _gguf_get_arr_str = _gguf_get_arr_strPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int, int)>(); + late final _llama_backend_initPtr = + _lookup>('llama_backend_init'); + late final _llama_backend_init = + _llama_backend_initPtr.asFunction(); - int gguf_get_n_tensors( - ffi.Pointer ctx, + void llama_batch_free( + llama_batch batch, ) { - return _gguf_get_n_tensors( - ctx, + return _llama_batch_free( + batch, ); } - late final _gguf_get_n_tensorsPtr = _lookup< - ffi.NativeFunction)>>( - 'gguf_get_n_tensors'); - late final _gguf_get_n_tensors = _gguf_get_n_tensorsPtr - .asFunction)>(); + late final _llama_batch_freePtr = + _lookup>( + 'llama_batch_free'); + late final _llama_batch_free = + _llama_batch_freePtr.asFunction(); - int gguf_find_tensor( - ffi.Pointer ctx, - ffi.Pointer name, + llama_batch llama_batch_get_one( + ffi.Pointer tokens, + int n_tokens, ) { - return _gguf_find_tensor( - ctx, - name, + return _llama_batch_get_one( + tokens, + n_tokens, ); } - late final _gguf_find_tensorPtr = _lookup< + late final _llama_batch_get_onePtr = _lookup< ffi.NativeFunction< - ffi.Int64 Function(ffi.Pointer, - ffi.Pointer)>>('gguf_find_tensor'); - late final _gguf_find_tensor = _gguf_find_tensorPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer)>(); + llama_batch Function( + ffi.Pointer, ffi.Int32)>>('llama_batch_get_one'); + late final _llama_batch_get_one = _llama_batch_get_onePtr + .asFunction, int)>(); - int gguf_get_tensor_offset( - ffi.Pointer ctx, - int tensor_id, + llama_batch llama_batch_init( + int n_tokens, + int embd, + int n_seq_max, ) { - return _gguf_get_tensor_offset( - ctx, - tensor_id, + return _llama_batch_init( + n_tokens, + embd, + n_seq_max, ); } - late final _gguf_get_tensor_offsetPtr = _lookup< + late final _llama_batch_initPtr = _lookup< ffi.NativeFunction< - ffi.Size Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_offset'); - late final _gguf_get_tensor_offset = _gguf_get_tensor_offsetPtr - .asFunction, int)>(); + llama_batch Function( + ffi.Int32, ffi.Int32, ffi.Int32)>>('llama_batch_init'); + late final _llama_batch_init = + _llama_batch_initPtr.asFunction(); - ffi.Pointer gguf_get_tensor_name( - ffi.Pointer ctx, - int tensor_id, + /// Apply chat template. Inspired by hf apply_chat_template() on python. + /// + /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template + /// @param tmpl A Jinja template to use for this chat. + /// @param chat Pointer to a list of multiple llama_chat_message + /// @param n_msg Number of llama_chat_message in this chat + /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. + /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages) + /// @param length The size of the allocated buffer + /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. + int llama_chat_apply_template( + ffi.Pointer tmpl, + ffi.Pointer chat, + int n_msg, + bool add_ass, + ffi.Pointer buf, + int length, ) { - return _gguf_get_tensor_name( - ctx, - tensor_id, + return _llama_chat_apply_template( + tmpl, + chat, + n_msg, + add_ass, + buf, + length, ); } - late final _gguf_get_tensor_namePtr = _lookup< + late final _llama_chat_apply_templatePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_name'); - late final _gguf_get_tensor_name = _gguf_get_tensor_namePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Bool, + ffi.Pointer, + ffi.Int32)>>('llama_chat_apply_template'); + late final _llama_chat_apply_template = + _llama_chat_apply_templatePtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, + int, bool, ffi.Pointer, int)>(); - ggml_type gguf_get_tensor_type( - ffi.Pointer ctx, - int tensor_id, + int llama_chat_builtin_templates( + ffi.Pointer> output, + int len, ) { - return ggml_type.fromValue(_gguf_get_tensor_type( - ctx, - tensor_id, - )); + return _llama_chat_builtin_templates( + output, + len, + ); } - late final _gguf_get_tensor_typePtr = _lookup< + late final _llama_chat_builtin_templatesPtr = _lookup< ffi.NativeFunction< - ffi.UnsignedInt Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_type'); - late final _gguf_get_tensor_type = _gguf_get_tensor_typePtr - .asFunction, int)>(); + ffi.Int32 Function(ffi.Pointer>, + ffi.Size)>>('llama_chat_builtin_templates'); + late final _llama_chat_builtin_templates = _llama_chat_builtin_templatesPtr + .asFunction>, int)>(); - int gguf_get_tensor_size( - ffi.Pointer ctx, - int tensor_id, - ) { - return _gguf_get_tensor_size( - ctx, - tensor_id, - ); + llama_context_params llama_context_default_params() { + return _llama_context_default_params(); } - late final _gguf_get_tensor_sizePtr = _lookup< - ffi.NativeFunction< - ffi.Size Function( - ffi.Pointer, ffi.Int64)>>('gguf_get_tensor_size'); - late final _gguf_get_tensor_size = _gguf_get_tensor_sizePtr - .asFunction, int)>(); + late final _llama_context_default_paramsPtr = + _lookup>( + 'llama_context_default_params'); + late final _llama_context_default_params = _llama_context_default_paramsPtr + .asFunction(); - int gguf_remove_key( - ffi.Pointer ctx, - ffi.Pointer key, + @Deprecated('use llama_state_get_data instead') + int llama_copy_state_data( + ffi.Pointer ctx, + ffi.Pointer dst, ) { - return _gguf_remove_key( + return _llama_copy_state_data( ctx, - key, + dst, ); } - late final _gguf_remove_keyPtr = _lookup< + late final _llama_copy_state_dataPtr = _lookup< ffi.NativeFunction< - ffi.Int64 Function(ffi.Pointer, - ffi.Pointer)>>('gguf_remove_key'); - late final _gguf_remove_key = _gguf_remove_keyPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Size Function(ffi.Pointer, + ffi.Pointer)>>('llama_copy_state_data'); + late final _llama_copy_state_data = _llama_copy_state_dataPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - void gguf_set_val_u8( - ffi.Pointer ctx, - ffi.Pointer key, - int val, + int llama_decode( + ffi.Pointer ctx, + llama_batch batch, ) { - return _gguf_set_val_u8( + return _llama_decode( ctx, - key, - val, + batch, ); } - late final _gguf_set_val_u8Ptr = _lookup< + late final _llama_decodePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Uint8)>>('gguf_set_val_u8'); - late final _gguf_set_val_u8 = _gguf_set_val_u8Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Int32 Function( + ffi.Pointer, llama_batch)>>('llama_decode'); + late final _llama_decode = _llama_decodePtr + .asFunction, llama_batch)>(); - void gguf_set_val_i8( - ffi.Pointer ctx, - ffi.Pointer key, - int val, + void llama_detach_threadpool( + ffi.Pointer ctx, ) { - return _gguf_set_val_i8( + return _llama_detach_threadpool( ctx, - key, - val, ); } - late final _gguf_set_val_i8Ptr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int8)>>('gguf_set_val_i8'); - late final _gguf_set_val_i8 = _gguf_set_val_i8Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _llama_detach_threadpoolPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_detach_threadpool'); + late final _llama_detach_threadpool = _llama_detach_threadpoolPtr + .asFunction)>(); - void gguf_set_val_u16( - ffi.Pointer ctx, - ffi.Pointer key, - int val, + /// @details Convert the provided tokens into text (inverse of llama_tokenize()). + /// @param text The char pointer must be large enough to hold the resulting text. + /// @return Returns the number of chars/bytes on success, no more than text_len_max. + /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned. + /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. + /// @param unparse_special If true, special tokens are rendered in the output. + int llama_detokenize( + ffi.Pointer vocab, + ffi.Pointer tokens, + int n_tokens, + ffi.Pointer text, + int text_len_max, + bool remove_special, + bool unparse_special, ) { - return _gguf_set_val_u16( - ctx, - key, - val, + return _llama_detokenize( + vocab, + tokens, + n_tokens, + text, + text_len_max, + remove_special, + unparse_special, ); } - late final _gguf_set_val_u16Ptr = _lookup< + late final _llama_detokenizePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Uint16)>>('gguf_set_val_u16'); - late final _gguf_set_val_u16 = _gguf_set_val_u16Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer, + ffi.Int32, + ffi.Pointer, + ffi.Int32, + ffi.Bool, + ffi.Bool)>>('llama_detokenize'); + late final _llama_detokenize = _llama_detokenizePtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int, + ffi.Pointer, int, bool, bool)>(); - void gguf_set_val_i16( - ffi.Pointer ctx, - ffi.Pointer key, - int val, + int llama_encode( + ffi.Pointer ctx, + llama_batch batch, ) { - return _gguf_set_val_i16( + return _llama_encode( ctx, - key, - val, + batch, ); } - late final _gguf_set_val_i16Ptr = _lookup< + late final _llama_encodePtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int16)>>('gguf_set_val_i16'); - late final _gguf_set_val_i16 = _gguf_set_val_i16Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Int32 Function( + ffi.Pointer, llama_batch)>>('llama_encode'); + late final _llama_encode = _llama_encodePtr + .asFunction, llama_batch)>(); - void gguf_set_val_u32( - ffi.Pointer ctx, - ffi.Pointer key, - int val, + ffi.Pointer llama_flash_attn_type_name( + llama_flash_attn_type flash_attn_type, ) { - return _gguf_set_val_u32( - ctx, - key, - val, + return _llama_flash_attn_type_name( + flash_attn_type.value, ); } - late final _gguf_set_val_u32Ptr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Uint32)>>('gguf_set_val_u32'); - late final _gguf_set_val_u32 = _gguf_set_val_u32Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _llama_flash_attn_type_namePtr = + _lookup Function(ffi.Int)>>( + 'llama_flash_attn_type_name'); + late final _llama_flash_attn_type_name = _llama_flash_attn_type_namePtr + .asFunction Function(int)>(); - void gguf_set_val_i32( - ffi.Pointer ctx, - ffi.Pointer key, - int val, + void llama_free( + ffi.Pointer ctx, ) { - return _gguf_set_val_i32( + return _llama_free( ctx, - key, - val, ); } - late final _gguf_set_val_i32Ptr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int32)>>('gguf_set_val_i32'); - late final _gguf_set_val_i32 = _gguf_set_val_i32Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _llama_freePtr = _lookup< + ffi.NativeFunction)>>( + 'llama_free'); + late final _llama_free = + _llama_freePtr.asFunction)>(); - void gguf_set_val_f32( - ffi.Pointer ctx, - ffi.Pointer key, - double val, + @Deprecated('use llama_model_free instead') + void llama_free_model( + ffi.Pointer model, ) { - return _gguf_set_val_f32( - ctx, - key, - val, + return _llama_free_model( + model, ); } - late final _gguf_set_val_f32Ptr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Float)>>('gguf_set_val_f32'); - late final _gguf_set_val_f32 = _gguf_set_val_f32Ptr.asFunction< - void Function( - ffi.Pointer, ffi.Pointer, double)>(); + late final _llama_free_modelPtr = + _lookup)>>( + 'llama_free_model'); + late final _llama_free_model = _llama_free_modelPtr + .asFunction)>(); - void gguf_set_val_u64( - ffi.Pointer ctx, - ffi.Pointer key, - int val, + ffi.Pointer llama_ftype_name( + llama_ftype ftype, ) { - return _gguf_set_val_u64( + return _llama_ftype_name( + ftype.value, + ); + } + + late final _llama_ftype_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'llama_ftype_name'); + late final _llama_ftype_name = + _llama_ftype_namePtr.asFunction Function(int)>(); + + ffi.Pointer llama_get_embeddings( + ffi.Pointer ctx, + ) { + return _llama_get_embeddings( ctx, - key, - val, ); } - late final _gguf_set_val_u64Ptr = _lookup< + late final _llama_get_embeddingsPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Uint64)>>('gguf_set_val_u64'); - late final _gguf_set_val_u64 = _gguf_set_val_u64Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer Function( + ffi.Pointer)>>('llama_get_embeddings'); + late final _llama_get_embeddings = _llama_get_embeddingsPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - void gguf_set_val_i64( - ffi.Pointer ctx, - ffi.Pointer key, - int val, + ffi.Pointer llama_get_embeddings_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_set_val_i64( + return _llama_get_embeddings_ith( ctx, - key, - val, + i, ); } - late final _gguf_set_val_i64Ptr = _lookup< + late final _llama_get_embeddings_ithPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Int64)>>('gguf_set_val_i64'); - late final _gguf_set_val_i64 = _gguf_set_val_i64Ptr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Int32)>>('llama_get_embeddings_ith'); + late final _llama_get_embeddings_ith = + _llama_get_embeddings_ithPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - void gguf_set_val_f64( - ffi.Pointer ctx, - ffi.Pointer key, - double val, + ffi.Pointer llama_get_embeddings_seq( + ffi.Pointer ctx, + int seq_id, ) { - return _gguf_set_val_f64( + return _llama_get_embeddings_seq( ctx, - key, - val, + seq_id, ); } - late final _gguf_set_val_f64Ptr = _lookup< + late final _llama_get_embeddings_seqPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Double)>>('gguf_set_val_f64'); - late final _gguf_set_val_f64 = _gguf_set_val_f64Ptr.asFunction< - void Function( - ffi.Pointer, ffi.Pointer, double)>(); + ffi.Pointer Function(ffi.Pointer, + llama_seq_id)>>('llama_get_embeddings_seq'); + late final _llama_get_embeddings_seq = + _llama_get_embeddings_seqPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - void gguf_set_val_bool( - ffi.Pointer ctx, - ffi.Pointer key, - bool val, + ffi.Pointer llama_get_logits( + ffi.Pointer ctx, ) { - return _gguf_set_val_bool( + return _llama_get_logits( ctx, - key, - val, ); } - late final _gguf_set_val_boolPtr = _lookup< + late final _llama_get_logitsPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Bool)>>('gguf_set_val_bool'); - late final _gguf_set_val_bool = _gguf_set_val_boolPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, bool)>(); + ffi.Pointer Function( + ffi.Pointer)>>('llama_get_logits'); + late final _llama_get_logits = _llama_get_logitsPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - void gguf_set_val_str( - ffi.Pointer ctx, - ffi.Pointer key, - ffi.Pointer val, + ffi.Pointer llama_get_logits_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_set_val_str( + return _llama_get_logits_ith( ctx, - key, - val, + i, ); } - late final _gguf_set_val_strPtr = _lookup< + late final _llama_get_logits_ithPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('gguf_set_val_str'); - late final _gguf_set_val_str = _gguf_set_val_strPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer, ffi.Int32)>>('llama_get_logits_ith'); + late final _llama_get_logits_ith = _llama_get_logits_ithPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - void gguf_set_arr_data( - ffi.Pointer ctx, - ffi.Pointer key, - gguf_type type, - ffi.Pointer data, - int n, + llama_memory_t llama_get_memory( + ffi.Pointer ctx, ) { - return _gguf_set_arr_data( + return _llama_get_memory( ctx, - key, - type.value, - data, - n, ); } - late final _gguf_set_arr_dataPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, - ffi.Pointer, - ffi.UnsignedInt, - ffi.Pointer, - ffi.Size)>>('gguf_set_arr_data'); - late final _gguf_set_arr_data = _gguf_set_arr_dataPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int, - ffi.Pointer, int)>(); + late final _llama_get_memoryPtr = _lookup< + ffi + .NativeFunction)>>( + 'llama_get_memory'); + late final _llama_get_memory = _llama_get_memoryPtr + .asFunction)>(); - void gguf_set_arr_str( - ffi.Pointer ctx, - ffi.Pointer key, - ffi.Pointer> data, - int n, + ffi.Pointer llama_get_model( + ffi.Pointer ctx, ) { - return _gguf_set_arr_str( + return _llama_get_model( ctx, - key, - data, - n, ); } - late final _gguf_set_arr_strPtr = _lookup< + late final _llama_get_modelPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer>, - ffi.Size)>>('gguf_set_arr_str'); - late final _gguf_set_arr_str = _gguf_set_arr_strPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer>, int)>(); + ffi.Pointer Function( + ffi.Pointer)>>('llama_get_model'); + late final _llama_get_model = _llama_get_modelPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - void gguf_set_kv( - ffi.Pointer ctx, - ffi.Pointer src, + int llama_get_sampled_candidates_count_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_set_kv( + return _llama_get_sampled_candidates_count_ith( ctx, - src, + i, ); } - late final _gguf_set_kvPtr = _lookup< + late final _llama_get_sampled_candidates_count_ithPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('gguf_set_kv'); - late final _gguf_set_kv = _gguf_set_kvPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Uint32 Function(ffi.Pointer, + ffi.Int32)>>('llama_get_sampled_candidates_count_ith'); + late final _llama_get_sampled_candidates_count_ith = + _llama_get_sampled_candidates_count_ithPtr + .asFunction, int)>(); - void gguf_add_tensor( - ffi.Pointer ctx, - ffi.Pointer tensor, + ffi.Pointer llama_get_sampled_candidates_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_add_tensor( + return _llama_get_sampled_candidates_ith( ctx, - tensor, + i, ); } - late final _gguf_add_tensorPtr = _lookup< + late final _llama_get_sampled_candidates_ithPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('gguf_add_tensor'); - late final _gguf_add_tensor = _gguf_add_tensorPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Int32)>>('llama_get_sampled_candidates_ith'); + late final _llama_get_sampled_candidates_ith = + _llama_get_sampled_candidates_ithPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - void gguf_set_tensor_type( - ffi.Pointer ctx, - ffi.Pointer name, - ggml_type type, + int llama_get_sampled_logits_count_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_set_tensor_type( + return _llama_get_sampled_logits_count_ith( ctx, - name, - type.value, + i, ); } - late final _gguf_set_tensor_typePtr = _lookup< + late final _llama_get_sampled_logits_count_ithPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.UnsignedInt)>>('gguf_set_tensor_type'); - late final _gguf_set_tensor_type = _gguf_set_tensor_typePtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, int)>(); + ffi.Uint32 Function(ffi.Pointer, + ffi.Int32)>>('llama_get_sampled_logits_count_ith'); + late final _llama_get_sampled_logits_count_ith = + _llama_get_sampled_logits_count_ithPtr + .asFunction, int)>(); - void gguf_set_tensor_data( - ffi.Pointer ctx, - ffi.Pointer name, - ffi.Pointer data, + ffi.Pointer llama_get_sampled_logits_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_set_tensor_data( + return _llama_get_sampled_logits_ith( ctx, - name, - data, + i, ); } - late final _gguf_set_tensor_dataPtr = _lookup< + late final _llama_get_sampled_logits_ithPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer)>>('gguf_set_tensor_data'); - late final _gguf_set_tensor_data = _gguf_set_tensor_dataPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Int32)>>('llama_get_sampled_logits_ith'); + late final _llama_get_sampled_logits_ith = + _llama_get_sampled_logits_ithPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - bool gguf_write_to_file_ptr( - ffi.Pointer ctx, - ffi.Pointer file, - bool only_meta, + int llama_get_sampled_probs_count_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_write_to_file_ptr( + return _llama_get_sampled_probs_count_ith( ctx, - file, - only_meta, + i, ); } - late final _gguf_write_to_file_ptrPtr = _lookup< + late final _llama_get_sampled_probs_count_ithPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, ffi.Pointer, - ffi.Bool)>>('gguf_write_to_file_ptr'); - late final _gguf_write_to_file_ptr = _gguf_write_to_file_ptrPtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer, bool)>(); + ffi.Uint32 Function(ffi.Pointer, + ffi.Int32)>>('llama_get_sampled_probs_count_ith'); + late final _llama_get_sampled_probs_count_ith = + _llama_get_sampled_probs_count_ithPtr + .asFunction, int)>(); - bool gguf_write_to_file( - ffi.Pointer ctx, - ffi.Pointer fname, - bool only_meta, + ffi.Pointer llama_get_sampled_probs_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_write_to_file( + return _llama_get_sampled_probs_ith( ctx, - fname, - only_meta, + i, ); } - late final _gguf_write_to_filePtr = _lookup< + late final _llama_get_sampled_probs_ithPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, ffi.Pointer, - ffi.Bool)>>('gguf_write_to_file'); - late final _gguf_write_to_file = _gguf_write_to_filePtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer, bool)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Int32)>>('llama_get_sampled_probs_ith'); + late final _llama_get_sampled_probs_ith = + _llama_get_sampled_probs_ithPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - int gguf_get_meta_size( - ffi.Pointer ctx, + int llama_get_sampled_token_ith( + ffi.Pointer ctx, + int i, ) { - return _gguf_get_meta_size( + return _llama_get_sampled_token_ith( ctx, + i, ); } - late final _gguf_get_meta_sizePtr = - _lookup)>>( - 'gguf_get_meta_size'); - late final _gguf_get_meta_size = _gguf_get_meta_sizePtr - .asFunction)>(); + late final _llama_get_sampled_token_ithPtr = _lookup< + ffi.NativeFunction< + llama_token Function(ffi.Pointer, + ffi.Int32)>>('llama_get_sampled_token_ith'); + late final _llama_get_sampled_token_ith = _llama_get_sampled_token_ithPtr + .asFunction, int)>(); - void gguf_get_meta_data( - ffi.Pointer ctx, - ffi.Pointer data, + @Deprecated('use llama_state_get_size instead') + int llama_get_state_size( + ffi.Pointer ctx, ) { - return _gguf_get_meta_data( + return _llama_get_state_size( ctx, - data, ); } - late final _gguf_get_meta_dataPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('gguf_get_meta_data'); - late final _gguf_get_meta_data = _gguf_get_meta_dataPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + late final _llama_get_state_sizePtr = _lookup< + ffi.NativeFunction)>>( + 'llama_get_state_size'); + late final _llama_get_state_size = _llama_get_state_sizePtr + .asFunction)>(); - ffi.Pointer llama_flash_attn_type_name( - llama_flash_attn_type flash_attn_type, + ffi.Pointer llama_init_from_model( + ffi.Pointer model, + llama_context_params params, ) { - return _llama_flash_attn_type_name( - flash_attn_type.value, + return _llama_init_from_model( + model, + params, ); } - late final _llama_flash_attn_type_namePtr = - _lookup Function(ffi.Int)>>( - 'llama_flash_attn_type_name'); - late final _llama_flash_attn_type_name = _llama_flash_attn_type_namePtr - .asFunction Function(int)>(); - - llama_model_params llama_model_default_params() { - return _llama_model_default_params(); - } - - late final _llama_model_default_paramsPtr = - _lookup>( - 'llama_model_default_params'); - late final _llama_model_default_params = _llama_model_default_paramsPtr - .asFunction(); - - llama_context_params llama_context_default_params() { - return _llama_context_default_params(); - } - - late final _llama_context_default_paramsPtr = - _lookup>( - 'llama_context_default_params'); - late final _llama_context_default_params = _llama_context_default_paramsPtr - .asFunction(); - - llama_sampler_chain_params llama_sampler_chain_default_params() { - return _llama_sampler_chain_default_params(); - } - - late final _llama_sampler_chain_default_paramsPtr = - _lookup>( - 'llama_sampler_chain_default_params'); - late final _llama_sampler_chain_default_params = - _llama_sampler_chain_default_paramsPtr - .asFunction(); - - llama_model_quantize_params llama_model_quantize_default_params() { - return _llama_model_quantize_default_params(); - } - - late final _llama_model_quantize_default_paramsPtr = - _lookup>( - 'llama_model_quantize_default_params'); - late final _llama_model_quantize_default_params = - _llama_model_quantize_default_paramsPtr - .asFunction(); - - void llama_backend_init() { - return _llama_backend_init(); - } - - late final _llama_backend_initPtr = - _lookup>('llama_backend_init'); - late final _llama_backend_init = - _llama_backend_initPtr.asFunction(); - - void llama_backend_free() { - return _llama_backend_free(); - } - - late final _llama_backend_freePtr = - _lookup>('llama_backend_free'); - late final _llama_backend_free = - _llama_backend_freePtr.asFunction(); - - void llama_numa_init( - ggml_numa_strategy numa, - ) { - return _llama_numa_init( - numa.value, - ); - } - - late final _llama_numa_initPtr = - _lookup>( - 'llama_numa_init'); - late final _llama_numa_init = - _llama_numa_initPtr.asFunction(); - - void llama_attach_threadpool( - ffi.Pointer ctx, - ggml_threadpool_t threadpool, - ggml_threadpool_t threadpool_batch, - ) { - return _llama_attach_threadpool( - ctx, - threadpool, - threadpool_batch, - ); - } - - late final _llama_attach_threadpoolPtr = _lookup< + late final _llama_init_from_modelPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ggml_threadpool_t, - ggml_threadpool_t)>>('llama_attach_threadpool'); - late final _llama_attach_threadpool = _llama_attach_threadpoolPtr.asFunction< - void Function( - ffi.Pointer, ggml_threadpool_t, ggml_threadpool_t)>(); + ffi.Pointer Function(ffi.Pointer, + llama_context_params)>>('llama_init_from_model'); + late final _llama_init_from_model = _llama_init_from_modelPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, llama_context_params)>(); - void llama_detach_threadpool( - ffi.Pointer ctx, + llama_load_mode llama_load_mode_from_str( + ffi.Pointer str, ) { - return _llama_detach_threadpool( - ctx, - ); + return llama_load_mode.fromValue(_llama_load_mode_from_str( + str, + )); } - late final _llama_detach_threadpoolPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_detach_threadpool'); - late final _llama_detach_threadpool = _llama_detach_threadpoolPtr - .asFunction)>(); + late final _llama_load_mode_from_strPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_load_mode_from_str'); + late final _llama_load_mode_from_str = _llama_load_mode_from_strPtr + .asFunction)>(); - ffi.Pointer llama_model_init_from_user( - ffi.Pointer metadata, - llama_model_set_tensor_data_t set_tensor_data, - ffi.Pointer set_tensor_data_ud, - llama_model_params params, + ffi.Pointer llama_load_mode_name( + llama_load_mode load_mode, ) { - return _llama_model_init_from_user( - metadata, - set_tensor_data, - set_tensor_data_ud, - params, + return _llama_load_mode_name( + load_mode.value, ); } - late final _llama_model_init_from_userPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - llama_model_set_tensor_data_t, - ffi.Pointer, - llama_model_params)>>('llama_model_init_from_user'); - late final _llama_model_init_from_user = - _llama_model_init_from_userPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - llama_model_set_tensor_data_t, - ffi.Pointer, - llama_model_params)>(); + late final _llama_load_mode_namePtr = _lookup< + ffi.NativeFunction Function(ffi.UnsignedInt)>>( + 'llama_load_mode_name'); + late final _llama_load_mode_name = _llama_load_mode_namePtr + .asFunction Function(int)>(); + @Deprecated('use llama_model_load_from_file instead') ffi.Pointer llama_load_model_from_file( ffi.Pointer path_model, llama_model_params params, @@ -14901,181 +15108,81 @@ class LlamaBindings { ffi.Pointer Function( ffi.Pointer, llama_model_params)>(); - ffi.Pointer llama_model_load_from_file( - ffi.Pointer path_model, - llama_model_params params, + @Deprecated('use llama_state_load_file instead') + bool llama_load_session_file( + ffi.Pointer ctx, + ffi.Pointer path_session, + ffi.Pointer tokens_out, + int n_token_capacity, + ffi.Pointer n_token_count_out, ) { - return _llama_model_load_from_file( - path_model, - params, + return _llama_load_session_file( + ctx, + path_session, + tokens_out, + n_token_capacity, + n_token_count_out, ); } - late final _llama_model_load_from_filePtr = _lookup< + late final _llama_load_session_filePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - llama_model_params)>>('llama_model_load_from_file'); - late final _llama_model_load_from_file = - _llama_model_load_from_filePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, llama_model_params)>(); + ffi.Bool Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Pointer)>>('llama_load_session_file'); + late final _llama_load_session_file = _llama_load_session_filePtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, int, ffi.Pointer)>(); - ffi.Pointer llama_model_load_from_file_ptr( - ffi.Pointer file, - llama_model_params params, + void llama_log_get( + ffi.Pointer log_callback, + ffi.Pointer> user_data, ) { - return _llama_model_load_from_file_ptr( - file, - params, + return _llama_log_get( + log_callback, + user_data, ); } - late final _llama_model_load_from_file_ptrPtr = _lookup< + late final _llama_log_getPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - llama_model_params)>>('llama_model_load_from_file_ptr'); - late final _llama_model_load_from_file_ptr = - _llama_model_load_from_file_ptrPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, llama_model_params)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer>)>>('llama_log_get'); + late final _llama_log_get = _llama_log_getPtr.asFunction< + void Function(ffi.Pointer, + ffi.Pointer>)>(); - ffi.Pointer llama_model_load_from_splits( - ffi.Pointer> paths, - int n_paths, - llama_model_params params, + void llama_log_set( + ggml_log_callback log_callback, + ffi.Pointer user_data, ) { - return _llama_model_load_from_splits( - paths, - n_paths, - params, + return _llama_log_set( + log_callback, + user_data, ); } - late final _llama_model_load_from_splitsPtr = _lookup< + late final _llama_log_setPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer>, - ffi.Size, llama_model_params)>>('llama_model_load_from_splits'); - late final _llama_model_load_from_splits = - _llama_model_load_from_splitsPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer>, int, llama_model_params)>(); + ffi.Void Function( + ggml_log_callback, ffi.Pointer)>>('llama_log_set'); + late final _llama_log_set = _llama_log_setPtr + .asFunction)>(); - void llama_model_save_to_file( - ffi.Pointer model, - ffi.Pointer path_model, - ) { - return _llama_model_save_to_file( - model, - path_model, - ); + int llama_max_devices() { + return _llama_max_devices(); } - late final _llama_model_save_to_filePtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('llama_model_save_to_file'); - late final _llama_model_save_to_file = - _llama_model_save_to_filePtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + late final _llama_max_devicesPtr = + _lookup>('llama_max_devices'); + late final _llama_max_devices = + _llama_max_devicesPtr.asFunction(); - void llama_free_model( - ffi.Pointer model, - ) { - return _llama_free_model( - model, - ); - } - - late final _llama_free_modelPtr = - _lookup)>>( - 'llama_free_model'); - late final _llama_free_model = _llama_free_modelPtr - .asFunction)>(); - - void llama_model_free( - ffi.Pointer model, - ) { - return _llama_model_free( - model, - ); - } - - late final _llama_model_freePtr = - _lookup)>>( - 'llama_model_free'); - late final _llama_model_free = _llama_model_freePtr - .asFunction)>(); - - ffi.Pointer llama_init_from_model( - ffi.Pointer model, - llama_context_params params, - ) { - return _llama_init_from_model( - model, - params, - ); - } - - late final _llama_init_from_modelPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - llama_context_params)>>('llama_init_from_model'); - late final _llama_init_from_model = _llama_init_from_modelPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, llama_context_params)>(); - - ffi.Pointer llama_new_context_with_model( - ffi.Pointer model, - llama_context_params params, - ) { - return _llama_new_context_with_model( - model, - params, - ); - } - - late final _llama_new_context_with_modelPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - llama_context_params)>>('llama_new_context_with_model'); - late final _llama_new_context_with_model = - _llama_new_context_with_modelPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, llama_context_params)>(); - - void llama_free( - ffi.Pointer ctx, - ) { - return _llama_free( - ctx, - ); - } - - late final _llama_freePtr = _lookup< - ffi.NativeFunction)>>( - 'llama_free'); - late final _llama_free = - _llama_freePtr.asFunction)>(); - - int llama_time_us() { - return _llama_time_us(); - } - - late final _llama_time_usPtr = - _lookup>('llama_time_us'); - late final _llama_time_us = _llama_time_usPtr.asFunction(); - - int llama_max_devices() { - return _llama_max_devices(); - } - - late final _llama_max_devicesPtr = - _lookup>('llama_max_devices'); - late final _llama_max_devices = - _llama_max_devicesPtr.asFunction(); - - int llama_max_parallel_sequences() { - return _llama_max_parallel_sequences(); + int llama_max_parallel_sequences() { + return _llama_max_parallel_sequences(); } late final _llama_max_parallel_sequencesPtr = @@ -15094,240 +15201,282 @@ class LlamaBindings { late final _llama_max_tensor_buft_overrides = _llama_max_tensor_buft_overridesPtr.asFunction(); - bool llama_supports_mmap() { - return _llama_supports_mmap(); + bool llama_memory_can_shift( + llama_memory_t mem, + ) { + return _llama_memory_can_shift( + mem, + ); } - late final _llama_supports_mmapPtr = - _lookup>('llama_supports_mmap'); - late final _llama_supports_mmap = - _llama_supports_mmapPtr.asFunction(); + late final _llama_memory_can_shiftPtr = + _lookup>( + 'llama_memory_can_shift'); + late final _llama_memory_can_shift = + _llama_memory_can_shiftPtr.asFunction(); - bool llama_supports_mlock() { - return _llama_supports_mlock(); + void llama_memory_clear( + llama_memory_t mem, + bool data, + ) { + return _llama_memory_clear( + mem, + data, + ); } - late final _llama_supports_mlockPtr = - _lookup>('llama_supports_mlock'); - late final _llama_supports_mlock = - _llama_supports_mlockPtr.asFunction(); + late final _llama_memory_clearPtr = + _lookup>( + 'llama_memory_clear'); + late final _llama_memory_clear = + _llama_memory_clearPtr.asFunction(); - bool llama_supports_gpu_offload() { - return _llama_supports_gpu_offload(); + void llama_memory_seq_add( + llama_memory_t mem, + int seq_id, + int p0, + int p1, + int delta, + ) { + return _llama_memory_seq_add( + mem, + seq_id, + p0, + p1, + delta, + ); } - late final _llama_supports_gpu_offloadPtr = - _lookup>( - 'llama_supports_gpu_offload'); - late final _llama_supports_gpu_offload = - _llama_supports_gpu_offloadPtr.asFunction(); + late final _llama_memory_seq_addPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(llama_memory_t, llama_seq_id, llama_pos, llama_pos, + llama_pos)>>('llama_memory_seq_add'); + late final _llama_memory_seq_add = _llama_memory_seq_addPtr + .asFunction(); - bool llama_supports_rpc() { - return _llama_supports_rpc(); + void llama_memory_seq_cp( + llama_memory_t mem, + int seq_id_src, + int seq_id_dst, + int p0, + int p1, + ) { + return _llama_memory_seq_cp( + mem, + seq_id_src, + seq_id_dst, + p0, + p1, + ); } - late final _llama_supports_rpcPtr = - _lookup>('llama_supports_rpc'); - late final _llama_supports_rpc = - _llama_supports_rpcPtr.asFunction(); + late final _llama_memory_seq_cpPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(llama_memory_t, llama_seq_id, llama_seq_id, + llama_pos, llama_pos)>>('llama_memory_seq_cp'); + late final _llama_memory_seq_cp = _llama_memory_seq_cpPtr + .asFunction(); - int llama_n_ctx( - ffi.Pointer ctx, + void llama_memory_seq_div( + llama_memory_t mem, + int seq_id, + int p0, + int p1, + int d, ) { - return _llama_n_ctx( - ctx, + return _llama_memory_seq_div( + mem, + seq_id, + p0, + p1, + d, ); } - late final _llama_n_ctxPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_n_ctx'); - late final _llama_n_ctx = - _llama_n_ctxPtr.asFunction)>(); + late final _llama_memory_seq_divPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(llama_memory_t, llama_seq_id, llama_pos, llama_pos, + ffi.Int)>>('llama_memory_seq_div'); + late final _llama_memory_seq_div = _llama_memory_seq_divPtr + .asFunction(); - int llama_n_ctx_seq( - ffi.Pointer ctx, + void llama_memory_seq_keep( + llama_memory_t mem, + int seq_id, ) { - return _llama_n_ctx_seq( - ctx, + return _llama_memory_seq_keep( + mem, + seq_id, ); } - late final _llama_n_ctx_seqPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_n_ctx_seq'); - late final _llama_n_ctx_seq = _llama_n_ctx_seqPtr - .asFunction)>(); + late final _llama_memory_seq_keepPtr = _lookup< + ffi.NativeFunction>( + 'llama_memory_seq_keep'); + late final _llama_memory_seq_keep = _llama_memory_seq_keepPtr + .asFunction(); - int llama_n_batch( - ffi.Pointer ctx, + int llama_memory_seq_pos_max( + llama_memory_t mem, + int seq_id, ) { - return _llama_n_batch( - ctx, + return _llama_memory_seq_pos_max( + mem, + seq_id, ); } - late final _llama_n_batchPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_n_batch'); - late final _llama_n_batch = - _llama_n_batchPtr.asFunction)>(); + late final _llama_memory_seq_pos_maxPtr = _lookup< + ffi.NativeFunction>( + 'llama_memory_seq_pos_max'); + late final _llama_memory_seq_pos_max = _llama_memory_seq_pos_maxPtr + .asFunction(); - int llama_n_ubatch( - ffi.Pointer ctx, + int llama_memory_seq_pos_min( + llama_memory_t mem, + int seq_id, ) { - return _llama_n_ubatch( - ctx, + return _llama_memory_seq_pos_min( + mem, + seq_id, ); } - late final _llama_n_ubatchPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_n_ubatch'); - late final _llama_n_ubatch = - _llama_n_ubatchPtr.asFunction)>(); - - int llama_n_seq_max( - ffi.Pointer ctx, - ) { - return _llama_n_seq_max( - ctx, - ); - } - - late final _llama_n_seq_maxPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_n_seq_max'); - late final _llama_n_seq_max = _llama_n_seq_maxPtr - .asFunction)>(); - - int llama_n_rs_seq( - ffi.Pointer ctx, - ) { - return _llama_n_rs_seq( - ctx, - ); - } - - late final _llama_n_rs_seqPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_n_rs_seq'); - late final _llama_n_rs_seq = - _llama_n_rs_seqPtr.asFunction)>(); + late final _llama_memory_seq_pos_minPtr = _lookup< + ffi.NativeFunction>( + 'llama_memory_seq_pos_min'); + late final _llama_memory_seq_pos_min = _llama_memory_seq_pos_minPtr + .asFunction(); - int llama_n_ctx_train( - ffi.Pointer model, + bool llama_memory_seq_rm( + llama_memory_t mem, + int seq_id, + int p0, + int p1, ) { - return _llama_n_ctx_train( - model, + return _llama_memory_seq_rm( + mem, + seq_id, + p0, + p1, ); } - late final _llama_n_ctx_trainPtr = - _lookup)>>( - 'llama_n_ctx_train'); - late final _llama_n_ctx_train = _llama_n_ctx_trainPtr - .asFunction)>(); + late final _llama_memory_seq_rmPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(llama_memory_t, llama_seq_id, llama_pos, + llama_pos)>>('llama_memory_seq_rm'); + late final _llama_memory_seq_rm = _llama_memory_seq_rmPtr + .asFunction(); - int llama_n_embd( + ffi.Pointer llama_model_chat_template( ffi.Pointer model, + ffi.Pointer name, ) { - return _llama_n_embd( + return _llama_model_chat_template( model, + name, ); } - late final _llama_n_embdPtr = - _lookup)>>( - 'llama_n_embd'); - late final _llama_n_embd = - _llama_n_embdPtr.asFunction)>(); + late final _llama_model_chat_templatePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('llama_model_chat_template'); + late final _llama_model_chat_template = + _llama_model_chat_templatePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - int llama_n_layer( + ffi.Pointer llama_model_cls_label( ffi.Pointer model, + int i, ) { - return _llama_n_layer( + return _llama_model_cls_label( model, + i, ); } - late final _llama_n_layerPtr = - _lookup)>>( - 'llama_n_layer'); - late final _llama_n_layer = - _llama_n_layerPtr.asFunction)>(); + late final _llama_model_cls_labelPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Uint32)>>('llama_model_cls_label'); + late final _llama_model_cls_label = _llama_model_cls_labelPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - int llama_n_head( + int llama_model_decoder_start_token( ffi.Pointer model, ) { - return _llama_n_head( + return _llama_model_decoder_start_token( model, ); } - late final _llama_n_headPtr = - _lookup)>>( - 'llama_n_head'); - late final _llama_n_head = - _llama_n_headPtr.asFunction)>(); + late final _llama_model_decoder_start_tokenPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_model_decoder_start_token'); + late final _llama_model_decoder_start_token = + _llama_model_decoder_start_tokenPtr + .asFunction)>(); - int llama_n_vocab( - ffi.Pointer vocab, - ) { - return _llama_n_vocab( - vocab, - ); + llama_model_params llama_model_default_params() { + return _llama_model_default_params(); } - late final _llama_n_vocabPtr = - _lookup)>>( - 'llama_n_vocab'); - late final _llama_n_vocab = - _llama_n_vocabPtr.asFunction)>(); + late final _llama_model_default_paramsPtr = + _lookup>( + 'llama_model_default_params'); + late final _llama_model_default_params = _llama_model_default_paramsPtr + .asFunction(); - ffi.Pointer llama_get_model( - ffi.Pointer ctx, + int llama_model_desc( + ffi.Pointer model, + ffi.Pointer buf, + int buf_size, ) { - return _llama_get_model( - ctx, + return _llama_model_desc( + model, + buf, + buf_size, ); } - late final _llama_get_modelPtr = _lookup< + late final _llama_model_descPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('llama_get_model'); - late final _llama_get_model = _llama_get_modelPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + ffi.Int32 Function(ffi.Pointer, ffi.Pointer, + ffi.Size)>>('llama_model_desc'); + late final _llama_model_desc = _llama_model_descPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int)>(); - llama_memory_t llama_get_memory( - ffi.Pointer ctx, + void llama_model_free( + ffi.Pointer model, ) { - return _llama_get_memory( - ctx, + return _llama_model_free( + model, ); } - late final _llama_get_memoryPtr = _lookup< - ffi - .NativeFunction)>>( - 'llama_get_memory'); - late final _llama_get_memory = _llama_get_memoryPtr - .asFunction)>(); + late final _llama_model_freePtr = + _lookup)>>( + 'llama_model_free'); + late final _llama_model_free = _llama_model_freePtr + .asFunction)>(); - llama_pooling_type llama_pooling_type$1( - ffi.Pointer ctx, + llama_ftype llama_model_ftype( + ffi.Pointer model, ) { - return llama_pooling_type.fromValue(_llama_pooling_type$1( - ctx, + return llama_ftype.fromValue(_llama_model_ftype( + model, )); } - late final _llama_pooling_type$1Ptr = - _lookup)>>( - 'llama_pooling_type'); - late final _llama_pooling_type$1 = _llama_pooling_type$1Ptr - .asFunction)>(); + late final _llama_model_ftypePtr = _lookup< + ffi + .NativeFunction)>>( + 'llama_model_ftype'); + late final _llama_model_ftype = _llama_model_ftypePtr + .asFunction)>(); ffi.Pointer llama_model_get_vocab( ffi.Pointer model, @@ -15344,243 +15493,203 @@ class LlamaBindings { late final _llama_model_get_vocab = _llama_model_get_vocabPtr.asFunction< ffi.Pointer Function(ffi.Pointer)>(); - llama_rope_type llama_model_rope_type( + bool llama_model_has_decoder( ffi.Pointer model, ) { - return llama_rope_type.fromValue(_llama_model_rope_type( + return _llama_model_has_decoder( model, - )); + ); } - late final _llama_model_rope_typePtr = - _lookup)>>( - 'llama_model_rope_type'); - late final _llama_model_rope_type = _llama_model_rope_typePtr - .asFunction)>(); + late final _llama_model_has_decoderPtr = + _lookup)>>( + 'llama_model_has_decoder'); + late final _llama_model_has_decoder = _llama_model_has_decoderPtr + .asFunction)>(); - int llama_model_n_ctx_train( + bool llama_model_has_encoder( ffi.Pointer model, ) { - return _llama_model_n_ctx_train( + return _llama_model_has_encoder( model, ); } - late final _llama_model_n_ctx_trainPtr = - _lookup)>>( - 'llama_model_n_ctx_train'); - late final _llama_model_n_ctx_train = _llama_model_n_ctx_trainPtr - .asFunction)>(); + late final _llama_model_has_encoderPtr = + _lookup)>>( + 'llama_model_has_encoder'); + late final _llama_model_has_encoder = _llama_model_has_encoderPtr + .asFunction)>(); - int llama_model_n_embd( + ffi.Pointer llama_model_init_from_user( + ffi.Pointer metadata, + llama_model_set_tensor_data_t set_tensor_data, + ffi.Pointer set_tensor_data_ud, + llama_model_params params, + ) { + return _llama_model_init_from_user( + metadata, + set_tensor_data, + set_tensor_data_ud, + params, + ); + } + + late final _llama_model_init_from_userPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + llama_model_set_tensor_data_t, + ffi.Pointer, + llama_model_params)>>('llama_model_init_from_user'); + late final _llama_model_init_from_user = + _llama_model_init_from_userPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + llama_model_set_tensor_data_t, + ffi.Pointer, + llama_model_params)>(); + + bool llama_model_is_diffusion( ffi.Pointer model, ) { - return _llama_model_n_embd( + return _llama_model_is_diffusion( model, ); } - late final _llama_model_n_embdPtr = - _lookup)>>( - 'llama_model_n_embd'); - late final _llama_model_n_embd = _llama_model_n_embdPtr - .asFunction)>(); + late final _llama_model_is_diffusionPtr = + _lookup)>>( + 'llama_model_is_diffusion'); + late final _llama_model_is_diffusion = _llama_model_is_diffusionPtr + .asFunction)>(); - int llama_model_n_embd_inp( + bool llama_model_is_hybrid( ffi.Pointer model, ) { - return _llama_model_n_embd_inp( + return _llama_model_is_hybrid( model, ); } - late final _llama_model_n_embd_inpPtr = - _lookup)>>( - 'llama_model_n_embd_inp'); - late final _llama_model_n_embd_inp = _llama_model_n_embd_inpPtr - .asFunction)>(); + late final _llama_model_is_hybridPtr = + _lookup)>>( + 'llama_model_is_hybrid'); + late final _llama_model_is_hybrid = _llama_model_is_hybridPtr + .asFunction)>(); - int llama_model_n_embd_out( + bool llama_model_is_recurrent( ffi.Pointer model, ) { - return _llama_model_n_embd_out( + return _llama_model_is_recurrent( model, ); } - late final _llama_model_n_embd_outPtr = - _lookup)>>( - 'llama_model_n_embd_out'); - late final _llama_model_n_embd_out = _llama_model_n_embd_outPtr - .asFunction)>(); + late final _llama_model_is_recurrentPtr = + _lookup)>>( + 'llama_model_is_recurrent'); + late final _llama_model_is_recurrent = _llama_model_is_recurrentPtr + .asFunction)>(); - int llama_model_n_layer( - ffi.Pointer model, + ffi.Pointer llama_model_load_from_file( + ffi.Pointer path_model, + llama_model_params params, ) { - return _llama_model_n_layer( - model, + return _llama_model_load_from_file( + path_model, + params, ); } - late final _llama_model_n_layerPtr = - _lookup)>>( - 'llama_model_n_layer'); - late final _llama_model_n_layer = _llama_model_n_layerPtr - .asFunction)>(); + late final _llama_model_load_from_filePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + llama_model_params)>>('llama_model_load_from_file'); + late final _llama_model_load_from_file = + _llama_model_load_from_filePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, llama_model_params)>(); - int llama_model_n_head( - ffi.Pointer model, + ffi.Pointer llama_model_load_from_file_ptr( + ffi.Pointer file, + llama_model_params params, ) { - return _llama_model_n_head( - model, + return _llama_model_load_from_file_ptr( + file, + params, ); } - late final _llama_model_n_headPtr = - _lookup)>>( - 'llama_model_n_head'); - late final _llama_model_n_head = _llama_model_n_headPtr - .asFunction)>(); + late final _llama_model_load_from_file_ptrPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + llama_model_params)>>('llama_model_load_from_file_ptr'); + late final _llama_model_load_from_file_ptr = + _llama_model_load_from_file_ptrPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, llama_model_params)>(); - int llama_model_n_head_kv( - ffi.Pointer model, + ffi.Pointer llama_model_load_from_splits( + ffi.Pointer> paths, + int n_paths, + llama_model_params params, ) { - return _llama_model_n_head_kv( - model, + return _llama_model_load_from_splits( + paths, + n_paths, + params, ); } - late final _llama_model_n_head_kvPtr = - _lookup)>>( - 'llama_model_n_head_kv'); - late final _llama_model_n_head_kv = _llama_model_n_head_kvPtr - .asFunction)>(); + late final _llama_model_load_from_splitsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer>, + ffi.Size, llama_model_params)>>('llama_model_load_from_splits'); + late final _llama_model_load_from_splits = + _llama_model_load_from_splitsPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer>, int, llama_model_params)>(); - int llama_model_n_swa( + int llama_model_meta_count( ffi.Pointer model, ) { - return _llama_model_n_swa( + return _llama_model_meta_count( model, ); } - late final _llama_model_n_swaPtr = + late final _llama_model_meta_countPtr = _lookup)>>( - 'llama_model_n_swa'); - late final _llama_model_n_swa = _llama_model_n_swaPtr - .asFunction)>(); - - double llama_model_rope_freq_scale_train( - ffi.Pointer model, - ) { - return _llama_model_rope_freq_scale_train( - model, - ); - } - - late final _llama_model_rope_freq_scale_trainPtr = - _lookup)>>( - 'llama_model_rope_freq_scale_train'); - late final _llama_model_rope_freq_scale_train = - _llama_model_rope_freq_scale_trainPtr - .asFunction)>(); - - int llama_model_n_cls_out( - ffi.Pointer model, - ) { - return _llama_model_n_cls_out( - model, - ); - } - - late final _llama_model_n_cls_outPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_model_n_cls_out'); - late final _llama_model_n_cls_out = _llama_model_n_cls_outPtr + 'llama_model_meta_count'); + late final _llama_model_meta_count = _llama_model_meta_countPtr .asFunction)>(); - ffi.Pointer llama_model_cls_label( + int llama_model_meta_key_by_index( ffi.Pointer model, int i, - ) { - return _llama_model_cls_label( - model, - i, - ); - } - - late final _llama_model_cls_labelPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Uint32)>>('llama_model_cls_label'); - late final _llama_model_cls_label = _llama_model_cls_labelPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); - - llama_vocab_type llama_vocab_type$1( - ffi.Pointer vocab, - ) { - return llama_vocab_type.fromValue(_llama_vocab_type$1( - vocab, - )); - } - - late final _llama_vocab_type$1Ptr = _lookup< - ffi - .NativeFunction)>>( - 'llama_vocab_type'); - late final _llama_vocab_type$1 = _llama_vocab_type$1Ptr - .asFunction)>(); - - int llama_vocab_n_tokens( - ffi.Pointer vocab, - ) { - return _llama_vocab_n_tokens( - vocab, - ); - } - - late final _llama_vocab_n_tokensPtr = - _lookup)>>( - 'llama_vocab_n_tokens'); - late final _llama_vocab_n_tokens = _llama_vocab_n_tokensPtr - .asFunction)>(); - - int llama_model_meta_val_str( - ffi.Pointer model, - ffi.Pointer key, ffi.Pointer buf, int buf_size, ) { - return _llama_model_meta_val_str( + return _llama_model_meta_key_by_index( model, - key, + i, buf, buf_size, ); } - late final _llama_model_meta_val_strPtr = _lookup< + late final _llama_model_meta_key_by_indexPtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, ffi.Size)>>('llama_model_meta_val_str'); - late final _llama_model_meta_val_str = - _llama_model_meta_val_strPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, int)>(); - - int llama_model_meta_count( - ffi.Pointer model, - ) { - return _llama_model_meta_count( - model, - ); - } - - late final _llama_model_meta_countPtr = - _lookup)>>( - 'llama_model_meta_count'); - late final _llama_model_meta_count = _llama_model_meta_countPtr - .asFunction)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Int32, + ffi.Pointer, + ffi.Size)>>('llama_model_meta_key_by_index'); + late final _llama_model_meta_key_by_index = + _llama_model_meta_key_by_indexPtr.asFunction< + int Function( + ffi.Pointer, int, ffi.Pointer, int)>(); ffi.Pointer llama_model_meta_key_str( llama_model_meta_key key, @@ -15596,31 +15705,28 @@ class LlamaBindings { late final _llama_model_meta_key_str = _llama_model_meta_key_strPtr .asFunction Function(int)>(); - int llama_model_meta_key_by_index( + int llama_model_meta_val_str( ffi.Pointer model, - int i, + ffi.Pointer key, ffi.Pointer buf, int buf_size, ) { - return _llama_model_meta_key_by_index( + return _llama_model_meta_val_str( model, - i, + key, buf, buf_size, ); } - late final _llama_model_meta_key_by_indexPtr = _lookup< + late final _llama_model_meta_val_strPtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Int32, - ffi.Pointer, - ffi.Size)>>('llama_model_meta_key_by_index'); - late final _llama_model_meta_key_by_index = - _llama_model_meta_key_by_indexPtr.asFunction< - int Function( - ffi.Pointer, int, ffi.Pointer, int)>(); + ffi.Int32 Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, ffi.Size)>>('llama_model_meta_val_str'); + late final _llama_model_meta_val_str = + _llama_model_meta_val_strPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, int)>(); int llama_model_meta_val_str_by_index( ffi.Pointer model, @@ -15648,156 +15754,159 @@ class LlamaBindings { int Function( ffi.Pointer, int, ffi.Pointer, int)>(); - int llama_model_desc( + int llama_model_n_cls_out( ffi.Pointer model, - ffi.Pointer buf, - int buf_size, ) { - return _llama_model_desc( + return _llama_model_n_cls_out( model, - buf, - buf_size, ); } - late final _llama_model_descPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function(ffi.Pointer, ffi.Pointer, - ffi.Size)>>('llama_model_desc'); - late final _llama_model_desc = _llama_model_descPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int)>(); - - int llama_model_size( + late final _llama_model_n_cls_outPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_model_n_cls_out'); + late final _llama_model_n_cls_out = _llama_model_n_cls_outPtr + .asFunction)>(); + + int llama_model_n_ctx_train( ffi.Pointer model, ) { - return _llama_model_size( + return _llama_model_n_ctx_train( model, ); } - late final _llama_model_sizePtr = _lookup< - ffi.NativeFunction)>>( - 'llama_model_size'); - late final _llama_model_size = - _llama_model_sizePtr.asFunction)>(); + late final _llama_model_n_ctx_trainPtr = + _lookup)>>( + 'llama_model_n_ctx_train'); + late final _llama_model_n_ctx_train = _llama_model_n_ctx_trainPtr + .asFunction)>(); - ffi.Pointer llama_model_chat_template( + int llama_model_n_embd( ffi.Pointer model, - ffi.Pointer name, ) { - return _llama_model_chat_template( + return _llama_model_n_embd( model, - name, ); } - late final _llama_model_chat_templatePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('llama_model_chat_template'); - late final _llama_model_chat_template = - _llama_model_chat_templatePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _llama_model_n_embdPtr = + _lookup)>>( + 'llama_model_n_embd'); + late final _llama_model_n_embd = _llama_model_n_embdPtr + .asFunction)>(); - int llama_model_n_params( + int llama_model_n_embd_inp( ffi.Pointer model, ) { - return _llama_model_n_params( + return _llama_model_n_embd_inp( model, ); } - late final _llama_model_n_paramsPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_model_n_params'); - late final _llama_model_n_params = _llama_model_n_paramsPtr + late final _llama_model_n_embd_inpPtr = + _lookup)>>( + 'llama_model_n_embd_inp'); + late final _llama_model_n_embd_inp = _llama_model_n_embd_inpPtr .asFunction)>(); - bool llama_model_has_encoder( + int llama_model_n_embd_out( ffi.Pointer model, ) { - return _llama_model_has_encoder( + return _llama_model_n_embd_out( model, ); } - late final _llama_model_has_encoderPtr = - _lookup)>>( - 'llama_model_has_encoder'); - late final _llama_model_has_encoder = _llama_model_has_encoderPtr - .asFunction)>(); + late final _llama_model_n_embd_outPtr = + _lookup)>>( + 'llama_model_n_embd_out'); + late final _llama_model_n_embd_out = _llama_model_n_embd_outPtr + .asFunction)>(); - bool llama_model_has_decoder( + int llama_model_n_head( ffi.Pointer model, ) { - return _llama_model_has_decoder( + return _llama_model_n_head( model, ); } - late final _llama_model_has_decoderPtr = - _lookup)>>( - 'llama_model_has_decoder'); - late final _llama_model_has_decoder = _llama_model_has_decoderPtr - .asFunction)>(); + late final _llama_model_n_headPtr = + _lookup)>>( + 'llama_model_n_head'); + late final _llama_model_n_head = _llama_model_n_headPtr + .asFunction)>(); - int llama_model_decoder_start_token( + int llama_model_n_head_kv( ffi.Pointer model, ) { - return _llama_model_decoder_start_token( + return _llama_model_n_head_kv( model, ); } - late final _llama_model_decoder_start_tokenPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_model_decoder_start_token'); - late final _llama_model_decoder_start_token = - _llama_model_decoder_start_tokenPtr - .asFunction)>(); + late final _llama_model_n_head_kvPtr = + _lookup)>>( + 'llama_model_n_head_kv'); + late final _llama_model_n_head_kv = _llama_model_n_head_kvPtr + .asFunction)>(); - bool llama_model_is_recurrent( + int llama_model_n_layer( ffi.Pointer model, ) { - return _llama_model_is_recurrent( + return _llama_model_n_layer( model, ); } - late final _llama_model_is_recurrentPtr = - _lookup)>>( - 'llama_model_is_recurrent'); - late final _llama_model_is_recurrent = _llama_model_is_recurrentPtr - .asFunction)>(); + late final _llama_model_n_layerPtr = + _lookup)>>( + 'llama_model_n_layer'); + late final _llama_model_n_layer = _llama_model_n_layerPtr + .asFunction)>(); - bool llama_model_is_hybrid( + int llama_model_n_layer_nextn( ffi.Pointer model, ) { - return _llama_model_is_hybrid( + return _llama_model_n_layer_nextn( model, ); } - late final _llama_model_is_hybridPtr = - _lookup)>>( - 'llama_model_is_hybrid'); - late final _llama_model_is_hybrid = _llama_model_is_hybridPtr - .asFunction)>(); + late final _llama_model_n_layer_nextnPtr = + _lookup)>>( + 'llama_model_n_layer_nextn'); + late final _llama_model_n_layer_nextn = _llama_model_n_layer_nextnPtr + .asFunction)>(); - bool llama_model_is_diffusion( + int llama_model_n_params( ffi.Pointer model, ) { - return _llama_model_is_diffusion( + return _llama_model_n_params( model, ); } - late final _llama_model_is_diffusionPtr = - _lookup)>>( - 'llama_model_is_diffusion'); - late final _llama_model_is_diffusion = _llama_model_is_diffusionPtr - .asFunction)>(); + late final _llama_model_n_paramsPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_model_n_params'); + late final _llama_model_n_params = _llama_model_n_paramsPtr + .asFunction)>(); + + int llama_model_n_swa( + ffi.Pointer model, + ) { + return _llama_model_n_swa( + model, + ); + } + + late final _llama_model_n_swaPtr = + _lookup)>>( + 'llama_model_n_swa'); + late final _llama_model_n_swa = _llama_model_n_swaPtr + .asFunction)>(); int llama_model_quantize( ffi.Pointer fname_inp, @@ -15820,1612 +15929,1837 @@ class LlamaBindings { int Function(ffi.Pointer, ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer llama_adapter_lora_init( + llama_model_quantize_params llama_model_quantize_default_params() { + return _llama_model_quantize_default_params(); + } + + late final _llama_model_quantize_default_paramsPtr = + _lookup>( + 'llama_model_quantize_default_params'); + late final _llama_model_quantize_default_params = + _llama_model_quantize_default_paramsPtr + .asFunction(); + + double llama_model_rope_freq_scale_train( ffi.Pointer model, - ffi.Pointer path_lora, ) { - return _llama_adapter_lora_init( + return _llama_model_rope_freq_scale_train( model, - path_lora, ); } - late final _llama_adapter_lora_initPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('llama_adapter_lora_init'); - late final _llama_adapter_lora_init = _llama_adapter_lora_initPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + late final _llama_model_rope_freq_scale_trainPtr = + _lookup)>>( + 'llama_model_rope_freq_scale_train'); + late final _llama_model_rope_freq_scale_train = + _llama_model_rope_freq_scale_trainPtr + .asFunction)>(); - int llama_adapter_meta_val_str( - ffi.Pointer adapter, - ffi.Pointer key, - ffi.Pointer buf, - int buf_size, + llama_rope_type llama_model_rope_type( + ffi.Pointer model, ) { - return _llama_adapter_meta_val_str( - adapter, - key, - buf, - buf_size, + return llama_rope_type.fromValue(_llama_model_rope_type( + model, + )); + } + + late final _llama_model_rope_typePtr = + _lookup)>>( + 'llama_model_rope_type'); + late final _llama_model_rope_type = _llama_model_rope_typePtr + .asFunction)>(); + + void llama_model_save_to_file( + ffi.Pointer model, + ffi.Pointer path_model, + ) { + return _llama_model_save_to_file( + model, + path_model, ); } - late final _llama_adapter_meta_val_strPtr = _lookup< + late final _llama_model_save_to_filePtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size)>>('llama_adapter_meta_val_str'); - late final _llama_adapter_meta_val_str = - _llama_adapter_meta_val_strPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, int)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('llama_model_save_to_file'); + late final _llama_model_save_to_file = + _llama_model_save_to_filePtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - int llama_adapter_meta_count( - ffi.Pointer adapter, + int llama_model_size( + ffi.Pointer model, ) { - return _llama_adapter_meta_count( - adapter, + return _llama_model_size( + model, ); } - late final _llama_adapter_meta_countPtr = _lookup< - ffi - .NativeFunction)>>( - 'llama_adapter_meta_count'); - late final _llama_adapter_meta_count = _llama_adapter_meta_countPtr - .asFunction)>(); + late final _llama_model_sizePtr = _lookup< + ffi.NativeFunction)>>( + 'llama_model_size'); + late final _llama_model_size = + _llama_model_sizePtr.asFunction)>(); - int llama_adapter_meta_key_by_index( - ffi.Pointer adapter, - int i, - ffi.Pointer buf, - int buf_size, + int llama_n_batch( + ffi.Pointer ctx, ) { - return _llama_adapter_meta_key_by_index( - adapter, - i, - buf, - buf_size, + return _llama_n_batch( + ctx, ); } - late final _llama_adapter_meta_key_by_indexPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Int32, - ffi.Pointer, - ffi.Size)>>('llama_adapter_meta_key_by_index'); - late final _llama_adapter_meta_key_by_index = - _llama_adapter_meta_key_by_indexPtr.asFunction< - int Function(ffi.Pointer, int, - ffi.Pointer, int)>(); + late final _llama_n_batchPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_n_batch'); + late final _llama_n_batch = + _llama_n_batchPtr.asFunction)>(); - int llama_adapter_meta_val_str_by_index( - ffi.Pointer adapter, - int i, - ffi.Pointer buf, - int buf_size, + int llama_n_ctx( + ffi.Pointer ctx, ) { - return _llama_adapter_meta_val_str_by_index( - adapter, - i, - buf, - buf_size, + return _llama_n_ctx( + ctx, ); } - late final _llama_adapter_meta_val_str_by_indexPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Int32, - ffi.Pointer, - ffi.Size)>>('llama_adapter_meta_val_str_by_index'); - late final _llama_adapter_meta_val_str_by_index = - _llama_adapter_meta_val_str_by_indexPtr.asFunction< - int Function(ffi.Pointer, int, - ffi.Pointer, int)>(); + late final _llama_n_ctxPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_n_ctx'); + late final _llama_n_ctx = + _llama_n_ctxPtr.asFunction)>(); - void llama_adapter_lora_free( - ffi.Pointer adapter, + int llama_n_ctx_seq( + ffi.Pointer ctx, ) { - return _llama_adapter_lora_free( - adapter, + return _llama_n_ctx_seq( + ctx, ); } - late final _llama_adapter_lora_freePtr = _lookup< - ffi - .NativeFunction)>>( - 'llama_adapter_lora_free'); - late final _llama_adapter_lora_free = _llama_adapter_lora_freePtr - .asFunction)>(); + late final _llama_n_ctx_seqPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_n_ctx_seq'); + late final _llama_n_ctx_seq = _llama_n_ctx_seqPtr + .asFunction)>(); - int llama_adapter_get_alora_n_invocation_tokens( - ffi.Pointer adapter, + @Deprecated('use llama_model_n_ctx_train instead') + int llama_n_ctx_train( + ffi.Pointer model, ) { - return _llama_adapter_get_alora_n_invocation_tokens( - adapter, + return _llama_n_ctx_train( + model, ); } - late final _llama_adapter_get_alora_n_invocation_tokensPtr = _lookup< - ffi.NativeFunction< - ffi.Uint64 Function(ffi.Pointer)>>( - 'llama_adapter_get_alora_n_invocation_tokens'); - late final _llama_adapter_get_alora_n_invocation_tokens = - _llama_adapter_get_alora_n_invocation_tokensPtr - .asFunction)>(); + late final _llama_n_ctx_trainPtr = + _lookup)>>( + 'llama_n_ctx_train'); + late final _llama_n_ctx_train = _llama_n_ctx_trainPtr + .asFunction)>(); - ffi.Pointer llama_adapter_get_alora_invocation_tokens( - ffi.Pointer adapter, + @Deprecated('use llama_model_n_embd instead') + int llama_n_embd( + ffi.Pointer model, ) { - return _llama_adapter_get_alora_invocation_tokens( - adapter, + return _llama_n_embd( + model, ); } - late final _llama_adapter_get_alora_invocation_tokensPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>( - 'llama_adapter_get_alora_invocation_tokens'); - late final _llama_adapter_get_alora_invocation_tokens = - _llama_adapter_get_alora_invocation_tokensPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + late final _llama_n_embdPtr = + _lookup)>>( + 'llama_n_embd'); + late final _llama_n_embd = + _llama_n_embdPtr.asFunction)>(); - int llama_set_adapters_lora( - ffi.Pointer ctx, - ffi.Pointer> adapters, - int n_adapters, - ffi.Pointer scales, + @Deprecated('use llama_model_n_head instead') + int llama_n_head( + ffi.Pointer model, ) { - return _llama_set_adapters_lora( - ctx, - adapters, - n_adapters, - scales, + return _llama_n_head( + model, ); } - late final _llama_set_adapters_loraPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer>, - ffi.Size, - ffi.Pointer)>>('llama_set_adapters_lora'); - late final _llama_set_adapters_lora = _llama_set_adapters_loraPtr.asFunction< - int Function( - ffi.Pointer, - ffi.Pointer>, - int, - ffi.Pointer)>(); + late final _llama_n_headPtr = + _lookup)>>( + 'llama_n_head'); + late final _llama_n_head = + _llama_n_headPtr.asFunction)>(); - int llama_set_adapter_cvec( - ffi.Pointer ctx, - ffi.Pointer data, - int len, - int n_embd, - int il_start, - int il_end, + @Deprecated('use llama_model_n_layer instead') + int llama_n_layer( + ffi.Pointer model, ) { - return _llama_set_adapter_cvec( - ctx, - data, - len, - n_embd, - il_start, - il_end, + return _llama_n_layer( + model, ); } - late final _llama_set_adapter_cvecPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Int32, - ffi.Int32, - ffi.Int32)>>('llama_set_adapter_cvec'); - late final _llama_set_adapter_cvec = _llama_set_adapter_cvecPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int, int, - int, int)>(); + late final _llama_n_layerPtr = + _lookup)>>( + 'llama_n_layer'); + late final _llama_n_layer = + _llama_n_layerPtr.asFunction)>(); - void llama_memory_clear( - llama_memory_t mem, - bool data, + int llama_n_rs_seq( + ffi.Pointer ctx, ) { - return _llama_memory_clear( - mem, - data, + return _llama_n_rs_seq( + ctx, ); } - late final _llama_memory_clearPtr = - _lookup>( - 'llama_memory_clear'); - late final _llama_memory_clear = - _llama_memory_clearPtr.asFunction(); + late final _llama_n_rs_seqPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_n_rs_seq'); + late final _llama_n_rs_seq = + _llama_n_rs_seqPtr.asFunction)>(); - bool llama_memory_seq_rm( - llama_memory_t mem, - int seq_id, - int p0, - int p1, + int llama_n_seq_max( + ffi.Pointer ctx, ) { - return _llama_memory_seq_rm( - mem, - seq_id, - p0, - p1, + return _llama_n_seq_max( + ctx, ); } - late final _llama_memory_seq_rmPtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function(llama_memory_t, llama_seq_id, llama_pos, - llama_pos)>>('llama_memory_seq_rm'); - late final _llama_memory_seq_rm = _llama_memory_seq_rmPtr - .asFunction(); + late final _llama_n_seq_maxPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_n_seq_max'); + late final _llama_n_seq_max = _llama_n_seq_maxPtr + .asFunction)>(); - void llama_memory_seq_cp( - llama_memory_t mem, - int seq_id_src, - int seq_id_dst, - int p0, - int p1, + int llama_n_threads( + ffi.Pointer ctx, ) { - return _llama_memory_seq_cp( - mem, - seq_id_src, - seq_id_dst, - p0, - p1, + return _llama_n_threads( + ctx, ); } - late final _llama_memory_seq_cpPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(llama_memory_t, llama_seq_id, llama_seq_id, - llama_pos, llama_pos)>>('llama_memory_seq_cp'); - late final _llama_memory_seq_cp = _llama_memory_seq_cpPtr - .asFunction(); + late final _llama_n_threadsPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_n_threads'); + late final _llama_n_threads = _llama_n_threadsPtr + .asFunction)>(); - void llama_memory_seq_keep( - llama_memory_t mem, - int seq_id, + int llama_n_threads_batch( + ffi.Pointer ctx, ) { - return _llama_memory_seq_keep( - mem, - seq_id, + return _llama_n_threads_batch( + ctx, ); } - late final _llama_memory_seq_keepPtr = _lookup< - ffi.NativeFunction>( - 'llama_memory_seq_keep'); - late final _llama_memory_seq_keep = _llama_memory_seq_keepPtr - .asFunction(); + late final _llama_n_threads_batchPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_n_threads_batch'); + late final _llama_n_threads_batch = _llama_n_threads_batchPtr + .asFunction)>(); - void llama_memory_seq_add( - llama_memory_t mem, - int seq_id, - int p0, - int p1, - int delta, + int llama_n_ubatch( + ffi.Pointer ctx, ) { - return _llama_memory_seq_add( - mem, - seq_id, - p0, - p1, - delta, + return _llama_n_ubatch( + ctx, ); } - late final _llama_memory_seq_addPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(llama_memory_t, llama_seq_id, llama_pos, llama_pos, - llama_pos)>>('llama_memory_seq_add'); - late final _llama_memory_seq_add = _llama_memory_seq_addPtr - .asFunction(); + late final _llama_n_ubatchPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_n_ubatch'); + late final _llama_n_ubatch = + _llama_n_ubatchPtr.asFunction)>(); - void llama_memory_seq_div( - llama_memory_t mem, - int seq_id, - int p0, - int p1, - int d, + @Deprecated('use llama_vocab_n_tokens instead') + int llama_n_vocab( + ffi.Pointer vocab, ) { - return _llama_memory_seq_div( - mem, - seq_id, - p0, - p1, - d, + return _llama_n_vocab( + vocab, ); } - late final _llama_memory_seq_divPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(llama_memory_t, llama_seq_id, llama_pos, llama_pos, - ffi.Int)>>('llama_memory_seq_div'); - late final _llama_memory_seq_div = _llama_memory_seq_divPtr - .asFunction(); + late final _llama_n_vocabPtr = + _lookup)>>( + 'llama_n_vocab'); + late final _llama_n_vocab = + _llama_n_vocabPtr.asFunction)>(); - int llama_memory_seq_pos_min( - llama_memory_t mem, - int seq_id, + @Deprecated('use llama_init_from_model instead') + ffi.Pointer llama_new_context_with_model( + ffi.Pointer model, + llama_context_params params, ) { - return _llama_memory_seq_pos_min( - mem, - seq_id, + return _llama_new_context_with_model( + model, + params, ); } - late final _llama_memory_seq_pos_minPtr = _lookup< - ffi.NativeFunction>( - 'llama_memory_seq_pos_min'); - late final _llama_memory_seq_pos_min = _llama_memory_seq_pos_minPtr - .asFunction(); + late final _llama_new_context_with_modelPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + llama_context_params)>>('llama_new_context_with_model'); + late final _llama_new_context_with_model = + _llama_new_context_with_modelPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, llama_context_params)>(); - int llama_memory_seq_pos_max( - llama_memory_t mem, - int seq_id, + void llama_numa_init( + ggml_numa_strategy numa, ) { - return _llama_memory_seq_pos_max( - mem, - seq_id, + return _llama_numa_init( + numa.value, ); } - late final _llama_memory_seq_pos_maxPtr = _lookup< - ffi.NativeFunction>( - 'llama_memory_seq_pos_max'); - late final _llama_memory_seq_pos_max = _llama_memory_seq_pos_maxPtr - .asFunction(); + late final _llama_numa_initPtr = + _lookup>( + 'llama_numa_init'); + late final _llama_numa_init = + _llama_numa_initPtr.asFunction(); - bool llama_memory_can_shift( - llama_memory_t mem, + void llama_opt_epoch( + ffi.Pointer lctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, + ggml_opt_result_t result_eval, + int idata_split, + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval, ) { - return _llama_memory_can_shift( - mem, + return _llama_opt_epoch( + lctx, + dataset, + result_train, + result_eval, + idata_split, + callback_train, + callback_eval, ); } - late final _llama_memory_can_shiftPtr = - _lookup>( - 'llama_memory_can_shift'); - late final _llama_memory_can_shift = - _llama_memory_can_shiftPtr.asFunction(); + late final _llama_opt_epochPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, + ggml_opt_dataset_t, + ggml_opt_result_t, + ggml_opt_result_t, + ffi.Int64, + ggml_opt_epoch_callback, + ggml_opt_epoch_callback)>>('llama_opt_epoch'); + late final _llama_opt_epoch = _llama_opt_epochPtr.asFunction< + void Function( + ffi.Pointer, + ggml_opt_dataset_t, + ggml_opt_result_t, + ggml_opt_result_t, + int, + ggml_opt_epoch_callback, + ggml_opt_epoch_callback)>(); - int llama_state_get_size( - ffi.Pointer ctx, + void llama_opt_init( + ffi.Pointer lctx, + ffi.Pointer model, + llama_opt_params lopt_params, ) { - return _llama_state_get_size( - ctx, + return _llama_opt_init( + lctx, + model, + lopt_params, ); } - late final _llama_state_get_sizePtr = _lookup< - ffi.NativeFunction)>>( - 'llama_state_get_size'); - late final _llama_state_get_size = _llama_state_get_sizePtr - .asFunction)>(); + late final _llama_opt_initPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, + ffi.Pointer, llama_opt_params)>>('llama_opt_init'); + late final _llama_opt_init = _llama_opt_initPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, + llama_opt_params)>(); - int llama_get_state_size( - ffi.Pointer ctx, + bool llama_opt_param_filter_all( + ffi.Pointer tensor, + ffi.Pointer userdata, ) { - return _llama_get_state_size( - ctx, + return _llama_opt_param_filter_all( + tensor, + userdata, ); } - late final _llama_get_state_sizePtr = _lookup< - ffi.NativeFunction)>>( - 'llama_get_state_size'); - late final _llama_get_state_size = _llama_get_state_sizePtr - .asFunction)>(); + late final _llama_opt_param_filter_allPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, + ffi.Pointer)>>('llama_opt_param_filter_all'); + late final _llama_opt_param_filter_all = + _llama_opt_param_filter_allPtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer)>(); - int llama_state_get_data( + llama_perf_context_data llama_perf_context( ffi.Pointer ctx, - ffi.Pointer dst, - int size, ) { - return _llama_state_get_data( + return _llama_perf_context( ctx, - dst, - size, ); } - late final _llama_state_get_dataPtr = _lookup< + late final _llama_perf_contextPtr = _lookup< ffi.NativeFunction< - ffi.Size Function(ffi.Pointer, ffi.Pointer, - ffi.Size)>>('llama_state_get_data'); - late final _llama_state_get_data = _llama_state_get_dataPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int)>(); + llama_perf_context_data Function( + ffi.Pointer)>>('llama_perf_context'); + late final _llama_perf_context = _llama_perf_contextPtr.asFunction< + llama_perf_context_data Function(ffi.Pointer)>(); - int llama_copy_state_data( + void llama_perf_context_print( ffi.Pointer ctx, - ffi.Pointer dst, ) { - return _llama_copy_state_data( + return _llama_perf_context_print( ctx, - dst, ); } - late final _llama_copy_state_dataPtr = _lookup< - ffi.NativeFunction< - ffi.Size Function(ffi.Pointer, - ffi.Pointer)>>('llama_copy_state_data'); - late final _llama_copy_state_data = _llama_copy_state_dataPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer)>(); + late final _llama_perf_context_printPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_perf_context_print'); + late final _llama_perf_context_print = _llama_perf_context_printPtr + .asFunction)>(); - int llama_state_set_data( + void llama_perf_context_reset( ffi.Pointer ctx, - ffi.Pointer src, - int size, ) { - return _llama_state_set_data( + return _llama_perf_context_reset( ctx, - src, - size, ); } - late final _llama_state_set_dataPtr = _lookup< - ffi.NativeFunction< - ffi.Size Function(ffi.Pointer, ffi.Pointer, - ffi.Size)>>('llama_state_set_data'); - late final _llama_state_set_data = _llama_state_set_dataPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int)>(); + late final _llama_perf_context_resetPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_perf_context_reset'); + late final _llama_perf_context_reset = _llama_perf_context_resetPtr + .asFunction)>(); - int llama_set_state_data( - ffi.Pointer ctx, - ffi.Pointer src, + llama_perf_sampler_data llama_perf_sampler( + ffi.Pointer chain, ) { - return _llama_set_state_data( - ctx, - src, + return _llama_perf_sampler( + chain, ); } - late final _llama_set_state_dataPtr = _lookup< + late final _llama_perf_samplerPtr = _lookup< ffi.NativeFunction< - ffi.Size Function(ffi.Pointer, - ffi.Pointer)>>('llama_set_state_data'); - late final _llama_set_state_data = _llama_set_state_dataPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer)>(); + llama_perf_sampler_data Function( + ffi.Pointer)>>('llama_perf_sampler'); + late final _llama_perf_sampler = _llama_perf_samplerPtr.asFunction< + llama_perf_sampler_data Function(ffi.Pointer)>(); - bool llama_state_load_file( - ffi.Pointer ctx, - ffi.Pointer path_session, - ffi.Pointer tokens_out, - int n_token_capacity, - ffi.Pointer n_token_count_out, + void llama_perf_sampler_print( + ffi.Pointer chain, ) { - return _llama_state_load_file( - ctx, - path_session, - tokens_out, - n_token_capacity, - n_token_count_out, + return _llama_perf_sampler_print( + chain, ); } - late final _llama_state_load_filePtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Pointer)>>('llama_state_load_file'); - late final _llama_state_load_file = _llama_state_load_filePtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, int, ffi.Pointer)>(); + late final _llama_perf_sampler_printPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_perf_sampler_print'); + late final _llama_perf_sampler_print = _llama_perf_sampler_printPtr + .asFunction)>(); - bool llama_load_session_file( - ffi.Pointer ctx, - ffi.Pointer path_session, - ffi.Pointer tokens_out, - int n_token_capacity, - ffi.Pointer n_token_count_out, + void llama_perf_sampler_reset( + ffi.Pointer chain, ) { - return _llama_load_session_file( - ctx, - path_session, - tokens_out, - n_token_capacity, - n_token_count_out, + return _llama_perf_sampler_reset( + chain, ); } - late final _llama_load_session_filePtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Pointer)>>('llama_load_session_file'); - late final _llama_load_session_file = _llama_load_session_filePtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, int, ffi.Pointer)>(); + late final _llama_perf_sampler_resetPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_perf_sampler_reset'); + late final _llama_perf_sampler_reset = _llama_perf_sampler_resetPtr + .asFunction)>(); - bool llama_state_save_file( + llama_pooling_type llama_pooling_type$1( ffi.Pointer ctx, - ffi.Pointer path_session, - ffi.Pointer tokens, - int n_token_count, ) { - return _llama_state_save_file( + return llama_pooling_type.fromValue(_llama_pooling_type( ctx, - path_session, - tokens, - n_token_count, + )); + } + + late final _llama_pooling_type$1Ptr = + _lookup)>>( + 'llama_pooling_type'); + late final _llama_pooling_type = _llama_pooling_type$1Ptr + .asFunction)>(); + + ffi.Pointer llama_print_system_info() { + return _llama_print_system_info(); + } + + late final _llama_print_system_infoPtr = + _lookup Function()>>( + 'llama_print_system_info'); + late final _llama_print_system_info = _llama_print_system_infoPtr + .asFunction Function()>(); + + void llama_sampler_accept( + ffi.Pointer smpl, + int token, + ) { + return _llama_sampler_accept( + smpl, + token, ); } - late final _llama_state_save_filePtr = _lookup< + late final _llama_sampler_acceptPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, ffi.Size)>>('llama_state_save_file'); - late final _llama_state_save_file = _llama_state_save_filePtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, int)>(); + ffi.Void Function(ffi.Pointer, + llama_token)>>('llama_sampler_accept'); + late final _llama_sampler_accept = _llama_sampler_acceptPtr + .asFunction, int)>(); - bool llama_save_session_file( - ffi.Pointer ctx, - ffi.Pointer path_session, - ffi.Pointer tokens, - int n_token_count, + void llama_sampler_apply( + ffi.Pointer smpl, + ffi.Pointer cur_p, ) { - return _llama_save_session_file( - ctx, - path_session, - tokens, - n_token_count, + return _llama_sampler_apply( + smpl, + cur_p, ); } - late final _llama_save_session_filePtr = _lookup< + late final _llama_sampler_applyPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, ffi.Size)>>('llama_save_session_file'); - late final _llama_save_session_file = _llama_save_session_filePtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer, - ffi.Pointer, int)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('llama_sampler_apply'); + late final _llama_sampler_apply = _llama_sampler_applyPtr.asFunction< + void Function( + ffi.Pointer, ffi.Pointer)>(); - int llama_state_seq_get_size( - ffi.Pointer ctx, - int seq_id, + void llama_sampler_chain_add( + ffi.Pointer chain, + ffi.Pointer smpl, ) { - return _llama_state_seq_get_size( - ctx, - seq_id, + return _llama_sampler_chain_add( + chain, + smpl, ); } - late final _llama_state_seq_get_sizePtr = _lookup< + late final _llama_sampler_chain_addPtr = _lookup< ffi.NativeFunction< - ffi.Size Function(ffi.Pointer, - llama_seq_id)>>('llama_state_seq_get_size'); - late final _llama_state_seq_get_size = _llama_state_seq_get_sizePtr - .asFunction, int)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('llama_sampler_chain_add'); + late final _llama_sampler_chain_add = _llama_sampler_chain_addPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - int llama_state_seq_get_data( - ffi.Pointer ctx, - ffi.Pointer dst, - int size, - int seq_id, + llama_sampler_chain_params llama_sampler_chain_default_params() { + return _llama_sampler_chain_default_params(); + } + + late final _llama_sampler_chain_default_paramsPtr = + _lookup>( + 'llama_sampler_chain_default_params'); + late final _llama_sampler_chain_default_params = + _llama_sampler_chain_default_paramsPtr + .asFunction(); + + ffi.Pointer llama_sampler_chain_get( + ffi.Pointer chain, + int i, ) { - return _llama_state_seq_get_data( - ctx, - dst, - size, - seq_id, + return _llama_sampler_chain_get( + chain, + i, ); } - late final _llama_state_seq_get_dataPtr = _lookup< + late final _llama_sampler_chain_getPtr = _lookup< ffi.NativeFunction< - ffi.Size Function(ffi.Pointer, ffi.Pointer, - ffi.Size, llama_seq_id)>>('llama_state_seq_get_data'); - late final _llama_state_seq_get_data = - _llama_state_seq_get_dataPtr.asFunction< - int Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Int32)>>('llama_sampler_chain_get'); + late final _llama_sampler_chain_get = _llama_sampler_chain_getPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - int llama_state_seq_set_data( - ffi.Pointer ctx, - ffi.Pointer src, - int size, - int dest_seq_id, + ffi.Pointer llama_sampler_chain_init( + llama_sampler_chain_params params, ) { - return _llama_state_seq_set_data( - ctx, - src, - size, - dest_seq_id, + return _llama_sampler_chain_init( + params, ); } - late final _llama_state_seq_set_dataPtr = _lookup< + late final _llama_sampler_chain_initPtr = _lookup< ffi.NativeFunction< - ffi.Size Function(ffi.Pointer, ffi.Pointer, - ffi.Size, llama_seq_id)>>('llama_state_seq_set_data'); - late final _llama_state_seq_set_data = - _llama_state_seq_set_dataPtr.asFunction< - int Function( - ffi.Pointer, ffi.Pointer, int, int)>(); + ffi.Pointer Function( + llama_sampler_chain_params)>>('llama_sampler_chain_init'); + late final _llama_sampler_chain_init = + _llama_sampler_chain_initPtr.asFunction< + ffi.Pointer Function(llama_sampler_chain_params)>(); - int llama_state_seq_save_file( - ffi.Pointer ctx, - ffi.Pointer filepath, - int seq_id, - ffi.Pointer tokens, - int n_token_count, + int llama_sampler_chain_n( + ffi.Pointer chain, ) { - return _llama_state_seq_save_file( - ctx, - filepath, - seq_id, - tokens, - n_token_count, + return _llama_sampler_chain_n( + chain, ); } - late final _llama_state_seq_save_filePtr = _lookup< - ffi.NativeFunction< - ffi.Size Function( - ffi.Pointer, - ffi.Pointer, - llama_seq_id, - ffi.Pointer, - ffi.Size)>>('llama_state_seq_save_file'); - late final _llama_state_seq_save_file = - _llama_state_seq_save_filePtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int, - ffi.Pointer, int)>(); + late final _llama_sampler_chain_nPtr = + _lookup)>>( + 'llama_sampler_chain_n'); + late final _llama_sampler_chain_n = _llama_sampler_chain_nPtr + .asFunction)>(); - int llama_state_seq_load_file( - ffi.Pointer ctx, - ffi.Pointer filepath, - int dest_seq_id, - ffi.Pointer tokens_out, - int n_token_capacity, - ffi.Pointer n_token_count_out, + ffi.Pointer llama_sampler_chain_remove( + ffi.Pointer chain, + int i, ) { - return _llama_state_seq_load_file( - ctx, - filepath, - dest_seq_id, - tokens_out, - n_token_capacity, - n_token_count_out, + return _llama_sampler_chain_remove( + chain, + i, ); } - late final _llama_state_seq_load_filePtr = _lookup< + late final _llama_sampler_chain_removePtr = _lookup< ffi.NativeFunction< - ffi.Size Function( - ffi.Pointer, - ffi.Pointer, - llama_seq_id, - ffi.Pointer, - ffi.Size, - ffi.Pointer)>>('llama_state_seq_load_file'); - late final _llama_state_seq_load_file = - _llama_state_seq_load_filePtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int, - ffi.Pointer, int, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Int32)>>('llama_sampler_chain_remove'); + late final _llama_sampler_chain_remove = + _llama_sampler_chain_removePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, int)>(); - int llama_state_seq_get_size_ext( - ffi.Pointer ctx, - int seq_id, - int flags, + ffi.Pointer llama_sampler_clone( + ffi.Pointer smpl, ) { - return _llama_state_seq_get_size_ext( - ctx, - seq_id, - flags, + return _llama_sampler_clone( + smpl, ); } - late final _llama_state_seq_get_size_extPtr = _lookup< + late final _llama_sampler_clonePtr = _lookup< ffi.NativeFunction< - ffi.Size Function(ffi.Pointer, llama_seq_id, - llama_state_seq_flags)>>('llama_state_seq_get_size_ext'); - late final _llama_state_seq_get_size_ext = _llama_state_seq_get_size_extPtr - .asFunction, int, int)>(); + ffi.Pointer Function( + ffi.Pointer)>>('llama_sampler_clone'); + late final _llama_sampler_clone = _llama_sampler_clonePtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - int llama_state_seq_get_data_ext( - ffi.Pointer ctx, - ffi.Pointer dst, - int size, - int seq_id, - int flags, + void llama_sampler_free( + ffi.Pointer smpl, ) { - return _llama_state_seq_get_data_ext( - ctx, - dst, - size, - seq_id, - flags, + return _llama_sampler_free( + smpl, ); } - late final _llama_state_seq_get_data_extPtr = _lookup< - ffi.NativeFunction< - ffi.Size Function( - ffi.Pointer, - ffi.Pointer, - ffi.Size, - llama_seq_id, - llama_state_seq_flags)>>('llama_state_seq_get_data_ext'); - late final _llama_state_seq_get_data_ext = - _llama_state_seq_get_data_extPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int, - int, int)>(); + late final _llama_sampler_freePtr = _lookup< + ffi.NativeFunction)>>( + 'llama_sampler_free'); + late final _llama_sampler_free = _llama_sampler_freePtr + .asFunction)>(); - int llama_state_seq_set_data_ext( - ffi.Pointer ctx, - ffi.Pointer src, - int size, - int dest_seq_id, - int flags, + int llama_sampler_get_seed( + ffi.Pointer smpl, ) { - return _llama_state_seq_set_data_ext( - ctx, - src, - size, - dest_seq_id, - flags, + return _llama_sampler_get_seed( + smpl, ); } - late final _llama_state_seq_set_data_extPtr = _lookup< - ffi.NativeFunction< - ffi.Size Function( - ffi.Pointer, - ffi.Pointer, - ffi.Size, - llama_seq_id, - llama_state_seq_flags)>>('llama_state_seq_set_data_ext'); - late final _llama_state_seq_set_data_ext = - _llama_state_seq_set_data_extPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int, - int, int)>(); + late final _llama_sampler_get_seedPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_sampler_get_seed'); + late final _llama_sampler_get_seed = _llama_sampler_get_seedPtr + .asFunction)>(); - llama_batch llama_batch_get_one( - ffi.Pointer tokens, - int n_tokens, + ffi.Pointer llama_sampler_init( + ffi.Pointer iface, + llama_sampler_context_t ctx, ) { - return _llama_batch_get_one( - tokens, - n_tokens, + return _llama_sampler_init( + iface, + ctx, ); } - late final _llama_batch_get_onePtr = _lookup< + late final _llama_sampler_initPtr = _lookup< ffi.NativeFunction< - llama_batch Function( - ffi.Pointer, ffi.Int32)>>('llama_batch_get_one'); - late final _llama_batch_get_one = _llama_batch_get_onePtr - .asFunction, int)>(); + ffi.Pointer Function(ffi.Pointer, + llama_sampler_context_t)>>('llama_sampler_init'); + late final _llama_sampler_init = _llama_sampler_initPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, llama_sampler_context_t)>(); - llama_batch llama_batch_init( - int n_tokens, - int embd, - int n_seq_max, + /// adaptive-p: select tokens near a configurable target probability over time. + /// + /// the adaptive-p sampler transforms the token probability distribution to favor tokens + /// that fall near a user-configurable probability target. + /// + /// internally, the sampler maintains an exponential moving average of the *ORIGINAL* + /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an + /// adapted target probability at each sampling step, thus maintaining the desired target + /// probability over time. + /// + /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last + /// in the sampler chain (like mirostat, dist, greedy). + /// + /// only mild truncation before this sampler is recommended. we suggest applying min-p + /// before adaptive-p as the only other active sampler in the chain. + /// + /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) + /// @param decay EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99) + /// @param seed RNG seed + /// + /// ref: https://github.com/ggml-org/llama.cpp/pull/17927 + ffi.Pointer llama_sampler_init_adaptive_p( + double target, + double decay, + int seed, ) { - return _llama_batch_init( - n_tokens, - embd, - n_seq_max, + return _llama_sampler_init_adaptive_p( + target, + decay, + seed, ); } - late final _llama_batch_initPtr = _lookup< + late final _llama_sampler_init_adaptive_pPtr = _lookup< ffi.NativeFunction< - llama_batch Function( - ffi.Int32, ffi.Int32, ffi.Int32)>>('llama_batch_init'); - late final _llama_batch_init = - _llama_batch_initPtr.asFunction(); + ffi.Pointer Function(ffi.Float, ffi.Float, + ffi.Uint32)>>('llama_sampler_init_adaptive_p'); + late final _llama_sampler_init_adaptive_p = _llama_sampler_init_adaptive_pPtr + .asFunction Function(double, double, int)>(); - void llama_batch_free( - llama_batch batch, + /// seed == LLAMA_DEFAULT_SEED to use a random seed. + ffi.Pointer llama_sampler_init_dist( + int seed, ) { - return _llama_batch_free( - batch, + return _llama_sampler_init_dist( + seed, ); } - late final _llama_batch_freePtr = - _lookup>( - 'llama_batch_free'); - late final _llama_batch_free = - _llama_batch_freePtr.asFunction(); + late final _llama_sampler_init_distPtr = _lookup< + ffi.NativeFunction Function(ffi.Uint32)>>( + 'llama_sampler_init_dist'); + late final _llama_sampler_init_dist = _llama_sampler_init_distPtr + .asFunction Function(int)>(); - int llama_encode( - ffi.Pointer ctx, - llama_batch batch, + /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 + ffi.Pointer llama_sampler_init_dry( + ffi.Pointer vocab, + int n_ctx_train, + double dry_multiplier, + double dry_base, + int dry_allowed_length, + int dry_penalty_last_n, + ffi.Pointer> seq_breakers, + int num_breakers, ) { - return _llama_encode( - ctx, - batch, + return _llama_sampler_init_dry( + vocab, + n_ctx_train, + dry_multiplier, + dry_base, + dry_allowed_length, + dry_penalty_last_n, + seq_breakers, + num_breakers, ); } - late final _llama_encodePtr = _lookup< + late final _llama_sampler_init_dryPtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, llama_batch)>>('llama_encode'); - late final _llama_encode = _llama_encodePtr - .asFunction, llama_batch)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Int32, + ffi.Float, + ffi.Float, + ffi.Int32, + ffi.Int32, + ffi.Pointer>, + ffi.Size)>>('llama_sampler_init_dry'); + late final _llama_sampler_init_dry = _llama_sampler_init_dryPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int, double, + double, int, int, ffi.Pointer>, int)>(); - int llama_decode( - ffi.Pointer ctx, - llama_batch batch, + /// @details Initializes a GBNF grammar, see grammars/README.md for details. + /// @param vocab The vocabulary that this grammar will be used with. + /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. + /// @param grammar_root The name of the start symbol for the grammar. + ffi.Pointer llama_sampler_init_grammar( + ffi.Pointer vocab, + ffi.Pointer grammar_str, + ffi.Pointer grammar_root, ) { - return _llama_decode( - ctx, - batch, + return _llama_sampler_init_grammar( + vocab, + grammar_str, + grammar_root, ); } - late final _llama_decodePtr = _lookup< + late final _llama_sampler_init_grammarPtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, llama_batch)>>('llama_decode'); - late final _llama_decode = _llama_decodePtr - .asFunction, llama_batch)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer)>>('llama_sampler_init_grammar'); + late final _llama_sampler_init_grammar = + _llama_sampler_init_grammarPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, ffi.Pointer)>(); - void llama_set_n_threads( - ffi.Pointer ctx, - int n_threads, - int n_threads_batch, + @Deprecated('use llama_sampler_init_grammar_lazy_patterns instead') + ffi.Pointer llama_sampler_init_grammar_lazy( + ffi.Pointer vocab, + ffi.Pointer grammar_str, + ffi.Pointer grammar_root, + ffi.Pointer> trigger_words, + int num_trigger_words, + ffi.Pointer trigger_tokens, + int num_trigger_tokens, ) { - return _llama_set_n_threads( - ctx, - n_threads, - n_threads_batch, + return _llama_sampler_init_grammar_lazy( + vocab, + grammar_str, + grammar_root, + trigger_words, + num_trigger_words, + trigger_tokens, + num_trigger_tokens, ); } - late final _llama_set_n_threadsPtr = _lookup< + late final _llama_sampler_init_grammar_lazyPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ffi.Int32, - ffi.Int32)>>('llama_set_n_threads'); - late final _llama_set_n_threads = _llama_set_n_threadsPtr - .asFunction, int, int)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>, + ffi.Size, + ffi.Pointer, + ffi.Size)>>('llama_sampler_init_grammar_lazy'); + late final _llama_sampler_init_grammar_lazy = + _llama_sampler_init_grammar_lazyPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>, + int, + ffi.Pointer, + int)>(); - int llama_n_threads( - ffi.Pointer ctx, + /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 + /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. + /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. + ffi.Pointer llama_sampler_init_grammar_lazy_patterns( + ffi.Pointer vocab, + ffi.Pointer grammar_str, + ffi.Pointer grammar_root, + ffi.Pointer> trigger_patterns, + int num_trigger_patterns, + ffi.Pointer trigger_tokens, + int num_trigger_tokens, ) { - return _llama_n_threads( - ctx, + return _llama_sampler_init_grammar_lazy_patterns( + vocab, + grammar_str, + grammar_root, + trigger_patterns, + num_trigger_patterns, + trigger_tokens, + num_trigger_tokens, ); } - late final _llama_n_threadsPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_n_threads'); - late final _llama_n_threads = _llama_n_threadsPtr - .asFunction)>(); + late final _llama_sampler_init_grammar_lazy_patternsPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>, + ffi.Size, + ffi.Pointer, + ffi.Size)>>('llama_sampler_init_grammar_lazy_patterns'); + late final _llama_sampler_init_grammar_lazy_patterns = + _llama_sampler_init_grammar_lazy_patternsPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>, + int, + ffi.Pointer, + int)>(); - int llama_n_threads_batch( - ffi.Pointer ctx, - ) { - return _llama_n_threads_batch( - ctx, - ); + ffi.Pointer llama_sampler_init_greedy() { + return _llama_sampler_init_greedy(); } - late final _llama_n_threads_batchPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_n_threads_batch'); - late final _llama_n_threads_batch = _llama_n_threads_batchPtr - .asFunction)>(); + late final _llama_sampler_init_greedyPtr = + _lookup Function()>>( + 'llama_sampler_init_greedy'); + late final _llama_sampler_init_greedy = _llama_sampler_init_greedyPtr + .asFunction Function()>(); - void llama_set_embeddings( - ffi.Pointer ctx, - bool embeddings, + ffi.Pointer llama_sampler_init_infill( + ffi.Pointer vocab, ) { - return _llama_set_embeddings( - ctx, - embeddings, + return _llama_sampler_init_infill( + vocab, ); } - late final _llama_set_embeddingsPtr = _lookup< + late final _llama_sampler_init_infillPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, ffi.Bool)>>('llama_set_embeddings'); - late final _llama_set_embeddings = _llama_set_embeddingsPtr - .asFunction, bool)>(); + ffi.Pointer Function( + ffi.Pointer)>>('llama_sampler_init_infill'); + late final _llama_sampler_init_infill = + _llama_sampler_init_infillPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - void llama_set_causal_attn( - ffi.Pointer ctx, - bool causal_attn, + ffi.Pointer llama_sampler_init_logit_bias( + int n_vocab, + int n_logit_bias, + ffi.Pointer logit_bias, ) { - return _llama_set_causal_attn( - ctx, - causal_attn, + return _llama_sampler_init_logit_bias( + n_vocab, + n_logit_bias, + logit_bias, ); } - late final _llama_set_causal_attnPtr = _lookup< + late final _llama_sampler_init_logit_biasPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, ffi.Bool)>>('llama_set_causal_attn'); - late final _llama_set_causal_attn = _llama_set_causal_attnPtr - .asFunction, bool)>(); + ffi.Pointer Function(ffi.Int32, ffi.Int32, + ffi.Pointer)>>('llama_sampler_init_logit_bias'); + late final _llama_sampler_init_logit_bias = + _llama_sampler_init_logit_biasPtr.asFunction< + ffi.Pointer Function( + int, int, ffi.Pointer)>(); - void llama_set_warmup( - ffi.Pointer ctx, - bool warmup, + /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 + ffi.Pointer llama_sampler_init_min_p( + double p, + int min_keep, ) { - return _llama_set_warmup( - ctx, - warmup, + return _llama_sampler_init_min_p( + p, + min_keep, ); } - late final _llama_set_warmupPtr = _lookup< + late final _llama_sampler_init_min_pPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, ffi.Bool)>>('llama_set_warmup'); - late final _llama_set_warmup = _llama_set_warmupPtr - .asFunction, bool)>(); + ffi.Pointer Function( + ffi.Float, ffi.Size)>>('llama_sampler_init_min_p'); + late final _llama_sampler_init_min_p = _llama_sampler_init_min_pPtr + .asFunction Function(double, int)>(); - void llama_set_abort_callback( - ffi.Pointer ctx, - ggml_abort_callback abort_callback, - ffi.Pointer abort_callback_data, + /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + ffi.Pointer llama_sampler_init_mirostat( + int n_vocab, + int seed, + double tau, + double eta, + int m, ) { - return _llama_set_abort_callback( - ctx, - abort_callback, - abort_callback_data, - ); - } + return _llama_sampler_init_mirostat( + n_vocab, + seed, + tau, + eta, + m, + ); + } - late final _llama_set_abort_callbackPtr = _lookup< + late final _llama_sampler_init_mirostatPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, ggml_abort_callback, - ffi.Pointer)>>('llama_set_abort_callback'); - late final _llama_set_abort_callback = - _llama_set_abort_callbackPtr.asFunction< - void Function(ffi.Pointer, ggml_abort_callback, - ffi.Pointer)>(); + ffi.Pointer Function(ffi.Int32, ffi.Uint32, ffi.Float, + ffi.Float, ffi.Int32)>>('llama_sampler_init_mirostat'); + late final _llama_sampler_init_mirostat = + _llama_sampler_init_mirostatPtr.asFunction< + ffi.Pointer Function(int, int, double, double, int)>(); - void llama_synchronize( - ffi.Pointer ctx, + /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + ffi.Pointer llama_sampler_init_mirostat_v2( + int seed, + double tau, + double eta, ) { - return _llama_synchronize( - ctx, + return _llama_sampler_init_mirostat_v2( + seed, + tau, + eta, ); } - late final _llama_synchronizePtr = _lookup< - ffi.NativeFunction)>>( - 'llama_synchronize'); - late final _llama_synchronize = _llama_synchronizePtr - .asFunction)>(); + late final _llama_sampler_init_mirostat_v2Ptr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Uint32, ffi.Float, + ffi.Float)>>('llama_sampler_init_mirostat_v2'); + late final _llama_sampler_init_mirostat_v2 = + _llama_sampler_init_mirostat_v2Ptr.asFunction< + ffi.Pointer Function(int, double, double)>(); - ffi.Pointer llama_get_logits( - ffi.Pointer ctx, + /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. + ffi.Pointer llama_sampler_init_penalties( + int penalty_last_n, + double penalty_repeat, + double penalty_freq, + double penalty_present, ) { - return _llama_get_logits( - ctx, + return _llama_sampler_init_penalties( + penalty_last_n, + penalty_repeat, + penalty_freq, + penalty_present, ); } - late final _llama_get_logitsPtr = _lookup< + late final _llama_sampler_init_penaltiesPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('llama_get_logits'); - late final _llama_get_logits = _llama_get_logitsPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + ffi.Pointer Function(ffi.Int32, ffi.Float, ffi.Float, + ffi.Float)>>('llama_sampler_init_penalties'); + late final _llama_sampler_init_penalties = + _llama_sampler_init_penaltiesPtr.asFunction< + ffi.Pointer Function(int, double, double, double)>(); - ffi.Pointer llama_get_logits_ith( - ffi.Pointer ctx, - int i, + /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf + ffi.Pointer llama_sampler_init_temp( + double t, ) { - return _llama_get_logits_ith( - ctx, - i, + return _llama_sampler_init_temp( + t, ); } - late final _llama_get_logits_ithPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Int32)>>('llama_get_logits_ith'); - late final _llama_get_logits_ith = _llama_get_logits_ithPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + late final _llama_sampler_init_tempPtr = _lookup< + ffi.NativeFunction Function(ffi.Float)>>( + 'llama_sampler_init_temp'); + late final _llama_sampler_init_temp = _llama_sampler_init_tempPtr + .asFunction Function(double)>(); - ffi.Pointer llama_get_embeddings( - ffi.Pointer ctx, + /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. + ffi.Pointer llama_sampler_init_temp_ext( + double t, + double delta, + double exponent, ) { - return _llama_get_embeddings( - ctx, + return _llama_sampler_init_temp_ext( + t, + delta, + exponent, ); } - late final _llama_get_embeddingsPtr = _lookup< + late final _llama_sampler_init_temp_extPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('llama_get_embeddings'); - late final _llama_get_embeddings = _llama_get_embeddingsPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Float, ffi.Float, ffi.Float)>>('llama_sampler_init_temp_ext'); + late final _llama_sampler_init_temp_ext = + _llama_sampler_init_temp_extPtr.asFunction< + ffi.Pointer Function(double, double, double)>(); - ffi.Pointer llama_get_embeddings_ith( - ffi.Pointer ctx, - int i, + /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// Setting k <= 0 makes this a noop + ffi.Pointer llama_sampler_init_top_k( + int k, ) { - return _llama_get_embeddings_ith( - ctx, - i, + return _llama_sampler_init_top_k( + k, ); } - late final _llama_get_embeddings_ithPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Int32)>>('llama_get_embeddings_ith'); - late final _llama_get_embeddings_ith = - _llama_get_embeddings_ithPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + late final _llama_sampler_init_top_kPtr = _lookup< + ffi.NativeFunction Function(ffi.Int32)>>( + 'llama_sampler_init_top_k'); + late final _llama_sampler_init_top_k = _llama_sampler_init_top_kPtr + .asFunction Function(int)>(); - ffi.Pointer llama_get_embeddings_seq( - ffi.Pointer ctx, - int seq_id, + /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 + ffi.Pointer llama_sampler_init_top_n_sigma( + double n, ) { - return _llama_get_embeddings_seq( - ctx, - seq_id, + return _llama_sampler_init_top_n_sigma( + n, ); } - late final _llama_get_embeddings_seqPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - llama_seq_id)>>('llama_get_embeddings_seq'); - late final _llama_get_embeddings_seq = - _llama_get_embeddings_seqPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + late final _llama_sampler_init_top_n_sigmaPtr = _lookup< + ffi.NativeFunction Function(ffi.Float)>>( + 'llama_sampler_init_top_n_sigma'); + late final _llama_sampler_init_top_n_sigma = + _llama_sampler_init_top_n_sigmaPtr + .asFunction Function(double)>(); - int llama_get_sampled_token_ith( - ffi.Pointer ctx, - int i, + /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + ffi.Pointer llama_sampler_init_top_p( + double p, + int min_keep, ) { - return _llama_get_sampled_token_ith( - ctx, - i, + return _llama_sampler_init_top_p( + p, + min_keep, ); } - late final _llama_get_sampled_token_ithPtr = _lookup< + late final _llama_sampler_init_top_pPtr = _lookup< ffi.NativeFunction< - llama_token Function(ffi.Pointer, - ffi.Int32)>>('llama_get_sampled_token_ith'); - late final _llama_get_sampled_token_ith = _llama_get_sampled_token_ithPtr - .asFunction, int)>(); + ffi.Pointer Function( + ffi.Float, ffi.Size)>>('llama_sampler_init_top_p'); + late final _llama_sampler_init_top_p = _llama_sampler_init_top_pPtr + .asFunction Function(double, int)>(); - ffi.Pointer llama_get_sampled_probs_ith( - ffi.Pointer ctx, - int i, + /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + ffi.Pointer llama_sampler_init_typical( + double p, + int min_keep, ) { - return _llama_get_sampled_probs_ith( - ctx, - i, + return _llama_sampler_init_typical( + p, + min_keep, ); } - late final _llama_get_sampled_probs_ithPtr = _lookup< + late final _llama_sampler_init_typicalPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Int32)>>('llama_get_sampled_probs_ith'); - late final _llama_get_sampled_probs_ith = - _llama_get_sampled_probs_ithPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Pointer Function( + ffi.Float, ffi.Size)>>('llama_sampler_init_typical'); + late final _llama_sampler_init_typical = _llama_sampler_init_typicalPtr + .asFunction Function(double, int)>(); - int llama_get_sampled_probs_count_ith( - ffi.Pointer ctx, - int i, + /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 + ffi.Pointer llama_sampler_init_xtc( + double p, + double t, + int min_keep, + int seed, ) { - return _llama_get_sampled_probs_count_ith( - ctx, - i, + return _llama_sampler_init_xtc( + p, + t, + min_keep, + seed, ); } - late final _llama_get_sampled_probs_count_ithPtr = _lookup< + late final _llama_sampler_init_xtcPtr = _lookup< ffi.NativeFunction< - ffi.Uint32 Function(ffi.Pointer, - ffi.Int32)>>('llama_get_sampled_probs_count_ith'); - late final _llama_get_sampled_probs_count_ith = - _llama_get_sampled_probs_count_ithPtr - .asFunction, int)>(); + ffi.Pointer Function(ffi.Float, ffi.Float, ffi.Size, + ffi.Uint32)>>('llama_sampler_init_xtc'); + late final _llama_sampler_init_xtc = _llama_sampler_init_xtcPtr.asFunction< + ffi.Pointer Function(double, double, int, int)>(); - ffi.Pointer llama_get_sampled_logits_ith( - ffi.Pointer ctx, - int i, + ffi.Pointer llama_sampler_name( + ffi.Pointer smpl, ) { - return _llama_get_sampled_logits_ith( - ctx, - i, + return _llama_sampler_name( + smpl, ); } - late final _llama_get_sampled_logits_ithPtr = _lookup< + late final _llama_sampler_namePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Int32)>>('llama_get_sampled_logits_ith'); - late final _llama_get_sampled_logits_ith = - _llama_get_sampled_logits_ithPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Pointer Function( + ffi.Pointer)>>('llama_sampler_name'); + late final _llama_sampler_name = _llama_sampler_namePtr + .asFunction Function(ffi.Pointer)>(); - int llama_get_sampled_logits_count_ith( - ffi.Pointer ctx, - int i, + void llama_sampler_reset( + ffi.Pointer smpl, ) { - return _llama_get_sampled_logits_count_ith( - ctx, - i, + return _llama_sampler_reset( + smpl, ); } - late final _llama_get_sampled_logits_count_ithPtr = _lookup< - ffi.NativeFunction< - ffi.Uint32 Function(ffi.Pointer, - ffi.Int32)>>('llama_get_sampled_logits_count_ith'); - late final _llama_get_sampled_logits_count_ith = - _llama_get_sampled_logits_count_ithPtr - .asFunction, int)>(); + late final _llama_sampler_resetPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_sampler_reset'); + late final _llama_sampler_reset = _llama_sampler_resetPtr + .asFunction)>(); - ffi.Pointer llama_get_sampled_candidates_ith( + int llama_sampler_sample( + ffi.Pointer smpl, ffi.Pointer ctx, - int i, + int idx, ) { - return _llama_get_sampled_candidates_ith( + return _llama_sampler_sample( + smpl, ctx, - i, + idx, ); } - late final _llama_get_sampled_candidates_ithPtr = _lookup< + late final _llama_sampler_samplePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Int32)>>('llama_get_sampled_candidates_ith'); - late final _llama_get_sampled_candidates_ith = - _llama_get_sampled_candidates_ithPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + llama_token Function(ffi.Pointer, + ffi.Pointer, ffi.Int32)>>('llama_sampler_sample'); + late final _llama_sampler_sample = _llama_sampler_samplePtr.asFunction< + int Function( + ffi.Pointer, ffi.Pointer, int)>(); - int llama_get_sampled_candidates_count_ith( + @Deprecated('use llama_state_save_file instead') + bool llama_save_session_file( ffi.Pointer ctx, - int i, + ffi.Pointer path_session, + ffi.Pointer tokens, + int n_token_count, ) { - return _llama_get_sampled_candidates_count_ith( + return _llama_save_session_file( ctx, - i, + path_session, + tokens, + n_token_count, ); } - late final _llama_get_sampled_candidates_count_ithPtr = _lookup< + late final _llama_save_session_filePtr = _lookup< ffi.NativeFunction< - ffi.Uint32 Function(ffi.Pointer, - ffi.Int32)>>('llama_get_sampled_candidates_count_ith'); - late final _llama_get_sampled_candidates_count_ith = - _llama_get_sampled_candidates_count_ithPtr - .asFunction, int)>(); + ffi.Bool Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, ffi.Size)>>('llama_save_session_file'); + late final _llama_save_session_file = _llama_save_session_filePtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, int)>(); - ffi.Pointer llama_vocab_get_text( - ffi.Pointer vocab, - int token, + void llama_set_abort_callback( + ffi.Pointer ctx, + ggml_abort_callback abort_callback, + ffi.Pointer abort_callback_data, ) { - return _llama_vocab_get_text( - vocab, - token, + return _llama_set_abort_callback( + ctx, + abort_callback, + abort_callback_data, ); } - late final _llama_vocab_get_textPtr = _lookup< + late final _llama_set_abort_callbackPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, llama_token)>>('llama_vocab_get_text'); - late final _llama_vocab_get_text = _llama_vocab_get_textPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Void Function(ffi.Pointer, ggml_abort_callback, + ffi.Pointer)>>('llama_set_abort_callback'); + late final _llama_set_abort_callback = + _llama_set_abort_callbackPtr.asFunction< + void Function(ffi.Pointer, ggml_abort_callback, + ffi.Pointer)>(); - double llama_vocab_get_score( - ffi.Pointer vocab, - int token, + int llama_set_adapter_cvec( + ffi.Pointer ctx, + ffi.Pointer data, + int len, + int n_embd, + int il_start, + int il_end, ) { - return _llama_vocab_get_score( - vocab, - token, + return _llama_set_adapter_cvec( + ctx, + data, + len, + n_embd, + il_start, + il_end, ); } - late final _llama_vocab_get_scorePtr = _lookup< + late final _llama_set_adapter_cvecPtr = _lookup< ffi.NativeFunction< - ffi.Float Function( - ffi.Pointer, llama_token)>>('llama_vocab_get_score'); - late final _llama_vocab_get_score = _llama_vocab_get_scorePtr - .asFunction, int)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Int32, + ffi.Int32, + ffi.Int32)>>('llama_set_adapter_cvec'); + late final _llama_set_adapter_cvec = _llama_set_adapter_cvecPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int, int, + int, int)>(); - llama_token_attr llama_vocab_get_attr( - ffi.Pointer vocab, - Dartllama_token token, + int llama_set_adapters_lora( + ffi.Pointer ctx, + ffi.Pointer> adapters, + int n_adapters, + ffi.Pointer scales, ) { - return llama_token_attr.fromValue(_llama_vocab_get_attr( - vocab, - token, - )); + return _llama_set_adapters_lora( + ctx, + adapters, + n_adapters, + scales, + ); } - late final _llama_vocab_get_attrPtr = _lookup< + late final _llama_set_adapters_loraPtr = _lookup< ffi.NativeFunction< - ffi.UnsignedInt Function( - ffi.Pointer, llama_token)>>('llama_vocab_get_attr'); - late final _llama_vocab_get_attr = _llama_vocab_get_attrPtr - .asFunction, int)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer>, + ffi.Size, + ffi.Pointer)>>('llama_set_adapters_lora'); + late final _llama_set_adapters_lora = _llama_set_adapters_loraPtr.asFunction< + int Function( + ffi.Pointer, + ffi.Pointer>, + int, + ffi.Pointer)>(); - bool llama_vocab_is_eog( - ffi.Pointer vocab, - int token, + void llama_set_causal_attn( + ffi.Pointer ctx, + bool causal_attn, ) { - return _llama_vocab_is_eog( - vocab, - token, + return _llama_set_causal_attn( + ctx, + causal_attn, ); } - late final _llama_vocab_is_eogPtr = _lookup< + late final _llama_set_causal_attnPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function( - ffi.Pointer, llama_token)>>('llama_vocab_is_eog'); - late final _llama_vocab_is_eog = _llama_vocab_is_eogPtr - .asFunction, int)>(); + ffi.Void Function( + ffi.Pointer, ffi.Bool)>>('llama_set_causal_attn'); + late final _llama_set_causal_attn = _llama_set_causal_attnPtr + .asFunction, bool)>(); - bool llama_vocab_is_control( - ffi.Pointer vocab, - int token, + void llama_set_embeddings( + ffi.Pointer ctx, + bool embeddings, ) { - return _llama_vocab_is_control( - vocab, - token, + return _llama_set_embeddings( + ctx, + embeddings, ); } - late final _llama_vocab_is_controlPtr = _lookup< + late final _llama_set_embeddingsPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, - llama_token)>>('llama_vocab_is_control'); - late final _llama_vocab_is_control = _llama_vocab_is_controlPtr - .asFunction, int)>(); + ffi.Void Function( + ffi.Pointer, ffi.Bool)>>('llama_set_embeddings'); + late final _llama_set_embeddings = _llama_set_embeddingsPtr + .asFunction, bool)>(); - int llama_vocab_bos( - ffi.Pointer vocab, + void llama_set_n_threads( + ffi.Pointer ctx, + int n_threads, + int n_threads_batch, ) { - return _llama_vocab_bos( - vocab, + return _llama_set_n_threads( + ctx, + n_threads, + n_threads_batch, ); } - late final _llama_vocab_bosPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_bos'); - late final _llama_vocab_bos = - _llama_vocab_bosPtr.asFunction)>(); + late final _llama_set_n_threadsPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, ffi.Int32, + ffi.Int32)>>('llama_set_n_threads'); + late final _llama_set_n_threads = _llama_set_n_threadsPtr + .asFunction, int, int)>(); - int llama_vocab_eos( - ffi.Pointer vocab, + bool llama_set_sampler( + ffi.Pointer ctx, + int seq_id, + ffi.Pointer smpl, ) { - return _llama_vocab_eos( - vocab, + return _llama_set_sampler( + ctx, + seq_id, + smpl, ); } - late final _llama_vocab_eosPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_eos'); - late final _llama_vocab_eos = - _llama_vocab_eosPtr.asFunction)>(); + late final _llama_set_samplerPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, llama_seq_id, + ffi.Pointer)>>('llama_set_sampler'); + late final _llama_set_sampler = _llama_set_samplerPtr.asFunction< + bool Function( + ffi.Pointer, int, ffi.Pointer)>(); - int llama_vocab_eot( - ffi.Pointer vocab, + @Deprecated('use llama_state_set_data instead') + int llama_set_state_data( + ffi.Pointer ctx, + ffi.Pointer src, ) { - return _llama_vocab_eot( - vocab, + return _llama_set_state_data( + ctx, + src, ); } - late final _llama_vocab_eotPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_eot'); - late final _llama_vocab_eot = - _llama_vocab_eotPtr.asFunction)>(); + late final _llama_set_state_dataPtr = _lookup< + ffi.NativeFunction< + ffi.Size Function(ffi.Pointer, + ffi.Pointer)>>('llama_set_state_data'); + late final _llama_set_state_data = _llama_set_state_dataPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - int llama_vocab_sep( - ffi.Pointer vocab, + @Deprecated( + 'user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]') + void llama_set_warmup( + ffi.Pointer ctx, + bool warmup, ) { - return _llama_vocab_sep( - vocab, + return _llama_set_warmup( + ctx, + warmup, ); } - late final _llama_vocab_sepPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_sep'); - late final _llama_vocab_sep = - _llama_vocab_sepPtr.asFunction)>(); + late final _llama_set_warmupPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, ffi.Bool)>>('llama_set_warmup'); + late final _llama_set_warmup = _llama_set_warmupPtr + .asFunction, bool)>(); - int llama_vocab_nl( - ffi.Pointer vocab, + /// @details Build a split GGUF final path for this chunk. + /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" + int llama_split_path( + ffi.Pointer split_path, + int maxlen, + ffi.Pointer path_prefix, + int split_no, + int split_count, ) { - return _llama_vocab_nl( - vocab, + return _llama_split_path( + split_path, + maxlen, + path_prefix, + split_no, + split_count, ); } - late final _llama_vocab_nlPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_nl'); - late final _llama_vocab_nl = - _llama_vocab_nlPtr.asFunction)>(); + late final _llama_split_pathPtr = _lookup< + ffi.NativeFunction< + ffi.Int32 Function( + ffi.Pointer, + ffi.Size, + ffi.Pointer, + ffi.Int32, + ffi.Int32)>>('llama_split_path'); + late final _llama_split_path = _llama_split_pathPtr.asFunction< + int Function( + ffi.Pointer, int, ffi.Pointer, int, int)>(); - int llama_vocab_pad( - ffi.Pointer vocab, + /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. + /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" + int llama_split_prefix( + ffi.Pointer split_prefix, + int maxlen, + ffi.Pointer split_path, + int split_no, + int split_count, ) { - return _llama_vocab_pad( - vocab, + return _llama_split_prefix( + split_prefix, + maxlen, + split_path, + split_no, + split_count, ); } - late final _llama_vocab_padPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_pad'); - late final _llama_vocab_pad = - _llama_vocab_padPtr.asFunction)>(); + late final _llama_split_prefixPtr = _lookup< + ffi.NativeFunction< + ffi.Int32 Function( + ffi.Pointer, + ffi.Size, + ffi.Pointer, + ffi.Int32, + ffi.Int32)>>('llama_split_prefix'); + late final _llama_split_prefix = _llama_split_prefixPtr.asFunction< + int Function( + ffi.Pointer, int, ffi.Pointer, int, int)>(); - int llama_vocab_mask( - ffi.Pointer vocab, + int llama_state_get_data( + ffi.Pointer ctx, + ffi.Pointer dst, + int size, ) { - return _llama_vocab_mask( - vocab, + return _llama_state_get_data( + ctx, + dst, + size, ); } - late final _llama_vocab_maskPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_mask'); - late final _llama_vocab_mask = - _llama_vocab_maskPtr.asFunction)>(); + late final _llama_state_get_dataPtr = _lookup< + ffi.NativeFunction< + ffi.Size Function(ffi.Pointer, ffi.Pointer, + ffi.Size)>>('llama_state_get_data'); + late final _llama_state_get_data = _llama_state_get_dataPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int)>(); - bool llama_vocab_get_add_bos( - ffi.Pointer vocab, + int llama_state_get_size( + ffi.Pointer ctx, ) { - return _llama_vocab_get_add_bos( - vocab, + return _llama_state_get_size( + ctx, ); } - late final _llama_vocab_get_add_bosPtr = - _lookup)>>( - 'llama_vocab_get_add_bos'); - late final _llama_vocab_get_add_bos = _llama_vocab_get_add_bosPtr - .asFunction)>(); + late final _llama_state_get_sizePtr = _lookup< + ffi.NativeFunction)>>( + 'llama_state_get_size'); + late final _llama_state_get_size = _llama_state_get_sizePtr + .asFunction)>(); - bool llama_vocab_get_add_eos( - ffi.Pointer vocab, + bool llama_state_load_file( + ffi.Pointer ctx, + ffi.Pointer path_session, + ffi.Pointer tokens_out, + int n_token_capacity, + ffi.Pointer n_token_count_out, ) { - return _llama_vocab_get_add_eos( - vocab, + return _llama_state_load_file( + ctx, + path_session, + tokens_out, + n_token_capacity, + n_token_count_out, ); } - late final _llama_vocab_get_add_eosPtr = - _lookup)>>( - 'llama_vocab_get_add_eos'); - late final _llama_vocab_get_add_eos = _llama_vocab_get_add_eosPtr - .asFunction)>(); + late final _llama_state_load_filePtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Pointer)>>('llama_state_load_file'); + late final _llama_state_load_file = _llama_state_load_filePtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, int, ffi.Pointer)>(); - bool llama_vocab_get_add_sep( - ffi.Pointer vocab, + bool llama_state_save_file( + ffi.Pointer ctx, + ffi.Pointer path_session, + ffi.Pointer tokens, + int n_token_count, ) { - return _llama_vocab_get_add_sep( - vocab, + return _llama_state_save_file( + ctx, + path_session, + tokens, + n_token_count, ); } - late final _llama_vocab_get_add_sepPtr = - _lookup)>>( - 'llama_vocab_get_add_sep'); - late final _llama_vocab_get_add_sep = _llama_vocab_get_add_sepPtr - .asFunction)>(); + late final _llama_state_save_filePtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, ffi.Size)>>('llama_state_save_file'); + late final _llama_state_save_file = _llama_state_save_filePtr.asFunction< + bool Function(ffi.Pointer, ffi.Pointer, + ffi.Pointer, int)>(); - int llama_vocab_fim_pre( - ffi.Pointer vocab, + int llama_state_seq_get_data( + ffi.Pointer ctx, + ffi.Pointer dst, + int size, + int seq_id, ) { - return _llama_vocab_fim_pre( - vocab, + return _llama_state_seq_get_data( + ctx, + dst, + size, + seq_id, ); } - late final _llama_vocab_fim_prePtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_fim_pre'); - late final _llama_vocab_fim_pre = _llama_vocab_fim_prePtr - .asFunction)>(); + late final _llama_state_seq_get_dataPtr = _lookup< + ffi.NativeFunction< + ffi.Size Function(ffi.Pointer, ffi.Pointer, + ffi.Size, llama_seq_id)>>('llama_state_seq_get_data'); + late final _llama_state_seq_get_data = + _llama_state_seq_get_dataPtr.asFunction< + int Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - int llama_vocab_fim_suf( - ffi.Pointer vocab, + int llama_state_seq_get_data_ext( + ffi.Pointer ctx, + ffi.Pointer dst, + int size, + int seq_id, + int flags, ) { - return _llama_vocab_fim_suf( - vocab, + return _llama_state_seq_get_data_ext( + ctx, + dst, + size, + seq_id, + flags, ); } - late final _llama_vocab_fim_sufPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_fim_suf'); - late final _llama_vocab_fim_suf = _llama_vocab_fim_sufPtr - .asFunction)>(); + late final _llama_state_seq_get_data_extPtr = _lookup< + ffi.NativeFunction< + ffi.Size Function( + ffi.Pointer, + ffi.Pointer, + ffi.Size, + llama_seq_id, + llama_state_seq_flags)>>('llama_state_seq_get_data_ext'); + late final _llama_state_seq_get_data_ext = + _llama_state_seq_get_data_extPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int, + int, int)>(); - int llama_vocab_fim_mid( - ffi.Pointer vocab, + int llama_state_seq_get_size( + ffi.Pointer ctx, + int seq_id, ) { - return _llama_vocab_fim_mid( - vocab, + return _llama_state_seq_get_size( + ctx, + seq_id, ); } - late final _llama_vocab_fim_midPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_fim_mid'); - late final _llama_vocab_fim_mid = _llama_vocab_fim_midPtr - .asFunction)>(); + late final _llama_state_seq_get_sizePtr = _lookup< + ffi.NativeFunction< + ffi.Size Function(ffi.Pointer, + llama_seq_id)>>('llama_state_seq_get_size'); + late final _llama_state_seq_get_size = _llama_state_seq_get_sizePtr + .asFunction, int)>(); - int llama_vocab_fim_pad( - ffi.Pointer vocab, + int llama_state_seq_get_size_ext( + ffi.Pointer ctx, + int seq_id, + int flags, ) { - return _llama_vocab_fim_pad( - vocab, + return _llama_state_seq_get_size_ext( + ctx, + seq_id, + flags, ); } - late final _llama_vocab_fim_padPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_fim_pad'); - late final _llama_vocab_fim_pad = _llama_vocab_fim_padPtr - .asFunction)>(); + late final _llama_state_seq_get_size_extPtr = _lookup< + ffi.NativeFunction< + ffi.Size Function(ffi.Pointer, llama_seq_id, + llama_state_seq_flags)>>('llama_state_seq_get_size_ext'); + late final _llama_state_seq_get_size_ext = _llama_state_seq_get_size_extPtr + .asFunction, int, int)>(); - int llama_vocab_fim_rep( - ffi.Pointer vocab, + int llama_state_seq_load_file( + ffi.Pointer ctx, + ffi.Pointer filepath, + int dest_seq_id, + ffi.Pointer tokens_out, + int n_token_capacity, + ffi.Pointer n_token_count_out, ) { - return _llama_vocab_fim_rep( - vocab, + return _llama_state_seq_load_file( + ctx, + filepath, + dest_seq_id, + tokens_out, + n_token_capacity, + n_token_count_out, ); } - late final _llama_vocab_fim_repPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_fim_rep'); - late final _llama_vocab_fim_rep = _llama_vocab_fim_repPtr - .asFunction)>(); + late final _llama_state_seq_load_filePtr = _lookup< + ffi.NativeFunction< + ffi.Size Function( + ffi.Pointer, + ffi.Pointer, + llama_seq_id, + ffi.Pointer, + ffi.Size, + ffi.Pointer)>>('llama_state_seq_load_file'); + late final _llama_state_seq_load_file = + _llama_state_seq_load_filePtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int, + ffi.Pointer, int, ffi.Pointer)>(); - int llama_vocab_fim_sep( - ffi.Pointer vocab, + int llama_state_seq_save_file( + ffi.Pointer ctx, + ffi.Pointer filepath, + int seq_id, + ffi.Pointer tokens, + int n_token_count, ) { - return _llama_vocab_fim_sep( - vocab, + return _llama_state_seq_save_file( + ctx, + filepath, + seq_id, + tokens, + n_token_count, ); } - late final _llama_vocab_fim_sepPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_vocab_fim_sep'); - late final _llama_vocab_fim_sep = _llama_vocab_fim_sepPtr - .asFunction)>(); + late final _llama_state_seq_save_filePtr = _lookup< + ffi.NativeFunction< + ffi.Size Function( + ffi.Pointer, + ffi.Pointer, + llama_seq_id, + ffi.Pointer, + ffi.Size)>>('llama_state_seq_save_file'); + late final _llama_state_seq_save_file = + _llama_state_seq_save_filePtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int, + ffi.Pointer, int)>(); - ffi.Pointer llama_token_get_text( - ffi.Pointer vocab, - int token, + int llama_state_seq_set_data( + ffi.Pointer ctx, + ffi.Pointer src, + int size, + int dest_seq_id, ) { - return _llama_token_get_text( - vocab, - token, + return _llama_state_seq_set_data( + ctx, + src, + size, + dest_seq_id, ); } - late final _llama_token_get_textPtr = _lookup< + late final _llama_state_seq_set_dataPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, llama_token)>>('llama_token_get_text'); - late final _llama_token_get_text = _llama_token_get_textPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Size Function(ffi.Pointer, ffi.Pointer, + ffi.Size, llama_seq_id)>>('llama_state_seq_set_data'); + late final _llama_state_seq_set_data = + _llama_state_seq_set_dataPtr.asFunction< + int Function( + ffi.Pointer, ffi.Pointer, int, int)>(); - double llama_token_get_score( - ffi.Pointer vocab, - int token, + int llama_state_seq_set_data_ext( + ffi.Pointer ctx, + ffi.Pointer src, + int size, + int dest_seq_id, + int flags, ) { - return _llama_token_get_score( - vocab, - token, + return _llama_state_seq_set_data_ext( + ctx, + src, + size, + dest_seq_id, + flags, ); } - late final _llama_token_get_scorePtr = _lookup< + late final _llama_state_seq_set_data_extPtr = _lookup< ffi.NativeFunction< - ffi.Float Function( - ffi.Pointer, llama_token)>>('llama_token_get_score'); - late final _llama_token_get_score = _llama_token_get_scorePtr - .asFunction, int)>(); + ffi.Size Function( + ffi.Pointer, + ffi.Pointer, + ffi.Size, + llama_seq_id, + llama_state_seq_flags)>>('llama_state_seq_set_data_ext'); + late final _llama_state_seq_set_data_ext = + _llama_state_seq_set_data_extPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int, + int, int)>(); - llama_token_attr llama_token_get_attr( - ffi.Pointer vocab, - Dartllama_token token, + int llama_state_set_data( + ffi.Pointer ctx, + ffi.Pointer src, + int size, ) { - return llama_token_attr.fromValue(_llama_token_get_attr( - vocab, - token, - )); + return _llama_state_set_data( + ctx, + src, + size, + ); } - late final _llama_token_get_attrPtr = _lookup< + late final _llama_state_set_dataPtr = _lookup< ffi.NativeFunction< - ffi.UnsignedInt Function( - ffi.Pointer, llama_token)>>('llama_token_get_attr'); - late final _llama_token_get_attr = _llama_token_get_attrPtr - .asFunction, int)>(); + ffi.Size Function(ffi.Pointer, ffi.Pointer, + ffi.Size)>>('llama_state_set_data'); + late final _llama_state_set_data = _llama_state_set_dataPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int)>(); - bool llama_token_is_eog( - ffi.Pointer vocab, - int token, - ) { - return _llama_token_is_eog( - vocab, - token, - ); + bool llama_supports_gpu_offload() { + return _llama_supports_gpu_offload(); } - late final _llama_token_is_eogPtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function( - ffi.Pointer, llama_token)>>('llama_token_is_eog'); - late final _llama_token_is_eog = _llama_token_is_eogPtr - .asFunction, int)>(); + late final _llama_supports_gpu_offloadPtr = + _lookup>( + 'llama_supports_gpu_offload'); + late final _llama_supports_gpu_offload = + _llama_supports_gpu_offloadPtr.asFunction(); - bool llama_token_is_control( - ffi.Pointer vocab, - int token, - ) { - return _llama_token_is_control( - vocab, - token, - ); + bool llama_supports_mlock() { + return _llama_supports_mlock(); } - late final _llama_token_is_controlPtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, - llama_token)>>('llama_token_is_control'); - late final _llama_token_is_control = _llama_token_is_controlPtr - .asFunction, int)>(); + late final _llama_supports_mlockPtr = + _lookup>('llama_supports_mlock'); + late final _llama_supports_mlock = + _llama_supports_mlockPtr.asFunction(); + + bool llama_supports_mmap() { + return _llama_supports_mmap(); + } + + late final _llama_supports_mmapPtr = + _lookup>('llama_supports_mmap'); + late final _llama_supports_mmap = + _llama_supports_mmapPtr.asFunction(); + + bool llama_supports_rpc() { + return _llama_supports_rpc(); + } + + late final _llama_supports_rpcPtr = + _lookup>('llama_supports_rpc'); + late final _llama_supports_rpc = + _llama_supports_rpcPtr.asFunction(); + + void llama_synchronize( + ffi.Pointer ctx, + ) { + return _llama_synchronize( + ctx, + ); + } + + late final _llama_synchronizePtr = _lookup< + ffi.NativeFunction)>>( + 'llama_synchronize'); + late final _llama_synchronize = _llama_synchronizePtr + .asFunction)>(); + + int llama_time_us() { + return _llama_time_us(); + } + + late final _llama_time_usPtr = + _lookup>('llama_time_us'); + late final _llama_time_us = _llama_time_usPtr.asFunction(); + @Deprecated('use llama_vocab_bos instead') int llama_token_bos( ffi.Pointer vocab, ) { @@ -17440,6 +17774,22 @@ class LlamaBindings { late final _llama_token_bos = _llama_token_bosPtr.asFunction)>(); + @Deprecated('use llama_vocab_cls instead') + int llama_token_cls( + ffi.Pointer vocab, + ) { + return _llama_token_cls( + vocab, + ); + } + + late final _llama_token_clsPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_token_cls'); + late final _llama_token_cls = + _llama_token_clsPtr.asFunction)>(); + + @Deprecated('use llama_vocab_eos instead') int llama_token_eos( ffi.Pointer vocab, ) { @@ -17454,6 +17804,7 @@ class LlamaBindings { late final _llama_token_eos = _llama_token_eosPtr.asFunction)>(); + @Deprecated('use llama_vocab_eot instead') int llama_token_eot( ffi.Pointer vocab, ) { @@ -17468,187 +17819,261 @@ class LlamaBindings { late final _llama_token_eot = _llama_token_eotPtr.asFunction)>(); - int llama_token_cls( + @Deprecated('use llama_vocab_fim_mid instead') + int llama_token_fim_mid( ffi.Pointer vocab, ) { - return _llama_token_cls( + return _llama_token_fim_mid( vocab, ); } - late final _llama_token_clsPtr = _lookup< + late final _llama_token_fim_midPtr = _lookup< ffi.NativeFunction)>>( - 'llama_token_cls'); - late final _llama_token_cls = - _llama_token_clsPtr.asFunction)>(); + 'llama_token_fim_mid'); + late final _llama_token_fim_mid = _llama_token_fim_midPtr + .asFunction)>(); - int llama_token_sep( + @Deprecated('use llama_vocab_fim_pad instead') + int llama_token_fim_pad( ffi.Pointer vocab, ) { - return _llama_token_sep( + return _llama_token_fim_pad( vocab, ); } - late final _llama_token_sepPtr = _lookup< + late final _llama_token_fim_padPtr = _lookup< ffi.NativeFunction)>>( - 'llama_token_sep'); - late final _llama_token_sep = - _llama_token_sepPtr.asFunction)>(); + 'llama_token_fim_pad'); + late final _llama_token_fim_pad = _llama_token_fim_padPtr + .asFunction)>(); - int llama_token_nl( + @Deprecated('use llama_vocab_fim_pre instead') + int llama_token_fim_pre( ffi.Pointer vocab, ) { - return _llama_token_nl( + return _llama_token_fim_pre( vocab, ); } - late final _llama_token_nlPtr = _lookup< + late final _llama_token_fim_prePtr = _lookup< ffi.NativeFunction)>>( - 'llama_token_nl'); - late final _llama_token_nl = - _llama_token_nlPtr.asFunction)>(); + 'llama_token_fim_pre'); + late final _llama_token_fim_pre = _llama_token_fim_prePtr + .asFunction)>(); - int llama_token_pad( + @Deprecated('use llama_vocab_fim_rep instead') + int llama_token_fim_rep( ffi.Pointer vocab, ) { - return _llama_token_pad( + return _llama_token_fim_rep( vocab, ); } - late final _llama_token_padPtr = _lookup< + late final _llama_token_fim_repPtr = _lookup< ffi.NativeFunction)>>( - 'llama_token_pad'); - late final _llama_token_pad = - _llama_token_padPtr.asFunction)>(); + 'llama_token_fim_rep'); + late final _llama_token_fim_rep = _llama_token_fim_repPtr + .asFunction)>(); - bool llama_add_bos_token( + @Deprecated('use llama_vocab_fim_sep instead') + int llama_token_fim_sep( ffi.Pointer vocab, ) { - return _llama_add_bos_token( + return _llama_token_fim_sep( vocab, ); } - late final _llama_add_bos_tokenPtr = - _lookup)>>( - 'llama_add_bos_token'); - late final _llama_add_bos_token = _llama_add_bos_tokenPtr - .asFunction)>(); + late final _llama_token_fim_sepPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_token_fim_sep'); + late final _llama_token_fim_sep = _llama_token_fim_sepPtr + .asFunction)>(); - bool llama_add_eos_token( + @Deprecated('use llama_vocab_fim_suf instead') + int llama_token_fim_suf( ffi.Pointer vocab, ) { - return _llama_add_eos_token( + return _llama_token_fim_suf( vocab, ); } - late final _llama_add_eos_tokenPtr = - _lookup)>>( - 'llama_add_eos_token'); - late final _llama_add_eos_token = _llama_add_eos_tokenPtr - .asFunction)>(); + late final _llama_token_fim_sufPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_token_fim_suf'); + late final _llama_token_fim_suf = _llama_token_fim_sufPtr + .asFunction)>(); - int llama_token_fim_pre( + @Deprecated('use llama_vocab_get_attr instead') + llama_token_attr llama_token_get_attr( ffi.Pointer vocab, + Dartllama_token token, ) { - return _llama_token_fim_pre( + return llama_token_attr.fromValue(_llama_token_get_attr( + vocab, + token, + )); + } + + late final _llama_token_get_attrPtr = _lookup< + ffi.NativeFunction< + ffi.UnsignedInt Function( + ffi.Pointer, llama_token)>>('llama_token_get_attr'); + late final _llama_token_get_attr = _llama_token_get_attrPtr + .asFunction, int)>(); + + @Deprecated('use llama_vocab_get_score instead') + double llama_token_get_score( + ffi.Pointer vocab, + int token, + ) { + return _llama_token_get_score( vocab, + token, ); } - late final _llama_token_fim_prePtr = _lookup< - ffi.NativeFunction)>>( - 'llama_token_fim_pre'); - late final _llama_token_fim_pre = _llama_token_fim_prePtr - .asFunction)>(); + late final _llama_token_get_scorePtr = _lookup< + ffi.NativeFunction< + ffi.Float Function( + ffi.Pointer, llama_token)>>('llama_token_get_score'); + late final _llama_token_get_score = _llama_token_get_scorePtr + .asFunction, int)>(); - int llama_token_fim_suf( + @Deprecated('use llama_vocab_get_text instead') + ffi.Pointer llama_token_get_text( ffi.Pointer vocab, + int token, ) { - return _llama_token_fim_suf( + return _llama_token_get_text( vocab, + token, ); } - late final _llama_token_fim_sufPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_token_fim_suf'); - late final _llama_token_fim_suf = _llama_token_fim_sufPtr - .asFunction)>(); + late final _llama_token_get_textPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, llama_token)>>('llama_token_get_text'); + late final _llama_token_get_text = _llama_token_get_textPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - int llama_token_fim_mid( + @Deprecated('use llama_vocab_is_control instead') + bool llama_token_is_control( ffi.Pointer vocab, + int token, ) { - return _llama_token_fim_mid( + return _llama_token_is_control( vocab, + token, ); } - late final _llama_token_fim_midPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_token_fim_mid'); - late final _llama_token_fim_mid = _llama_token_fim_midPtr - .asFunction)>(); + late final _llama_token_is_controlPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, + llama_token)>>('llama_token_is_control'); + late final _llama_token_is_control = _llama_token_is_controlPtr + .asFunction, int)>(); - int llama_token_fim_pad( + @Deprecated('use llama_vocab_is_eog instead') + bool llama_token_is_eog( ffi.Pointer vocab, + int token, ) { - return _llama_token_fim_pad( + return _llama_token_is_eog( vocab, + token, ); } - late final _llama_token_fim_padPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_token_fim_pad'); - late final _llama_token_fim_pad = _llama_token_fim_padPtr - .asFunction)>(); + late final _llama_token_is_eogPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function( + ffi.Pointer, llama_token)>>('llama_token_is_eog'); + late final _llama_token_is_eog = _llama_token_is_eogPtr + .asFunction, int)>(); - int llama_token_fim_rep( + @Deprecated('use llama_vocab_nl instead') + int llama_token_nl( ffi.Pointer vocab, ) { - return _llama_token_fim_rep( + return _llama_token_nl( vocab, ); } - late final _llama_token_fim_repPtr = _lookup< + late final _llama_token_nlPtr = _lookup< ffi.NativeFunction)>>( - 'llama_token_fim_rep'); - late final _llama_token_fim_rep = _llama_token_fim_repPtr - .asFunction)>(); + 'llama_token_nl'); + late final _llama_token_nl = + _llama_token_nlPtr.asFunction)>(); - int llama_token_fim_sep( + @Deprecated('use llama_vocab_pad instead') + int llama_token_pad( ffi.Pointer vocab, ) { - return _llama_token_fim_sep( + return _llama_token_pad( vocab, ); } - late final _llama_token_fim_sepPtr = _lookup< + late final _llama_token_padPtr = _lookup< ffi.NativeFunction)>>( - 'llama_token_fim_sep'); - late final _llama_token_fim_sep = _llama_token_fim_sepPtr - .asFunction)>(); + 'llama_token_pad'); + late final _llama_token_pad = + _llama_token_padPtr.asFunction)>(); - int llama_vocab_cls( + @Deprecated('use llama_vocab_sep instead') + int llama_token_sep( ffi.Pointer vocab, ) { - return _llama_vocab_cls( + return _llama_token_sep( vocab, ); } - late final _llama_vocab_clsPtr = _lookup< + late final _llama_token_sepPtr = _lookup< ffi.NativeFunction)>>( - 'llama_vocab_cls'); - late final _llama_vocab_cls = - _llama_vocab_clsPtr.asFunction)>(); + 'llama_token_sep'); + late final _llama_token_sep = + _llama_token_sepPtr.asFunction)>(); + + int llama_token_to_piece( + ffi.Pointer vocab, + int token, + ffi.Pointer buf, + int length, + int lstrip, + bool special, + ) { + return _llama_token_to_piece( + vocab, + token, + buf, + length, + lstrip, + special, + ); + } + + late final _llama_token_to_piecePtr = _lookup< + ffi.NativeFunction< + ffi.Int32 Function( + ffi.Pointer, + llama_token, + ffi.Pointer, + ffi.Int32, + ffi.Int32, + ffi.Bool)>>('llama_token_to_piece'); + late final _llama_token_to_piece = _llama_token_to_piecePtr.asFunction< + int Function(ffi.Pointer, int, ffi.Pointer, int, + int, bool)>(); /// @details Convert the provided text into tokens. /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. @@ -17692,1565 +18117,1375 @@ class LlamaBindings { int Function(ffi.Pointer, ffi.Pointer, int, ffi.Pointer, int, bool, bool)>(); - int llama_token_to_piece( + int llama_vocab_bos( ffi.Pointer vocab, - int token, - ffi.Pointer buf, - int length, - int lstrip, - bool special, ) { - return _llama_token_to_piece( + return _llama_vocab_bos( vocab, - token, - buf, - length, - lstrip, - special, ); } - late final _llama_token_to_piecePtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - llama_token, - ffi.Pointer, - ffi.Int32, - ffi.Int32, - ffi.Bool)>>('llama_token_to_piece'); - late final _llama_token_to_piece = _llama_token_to_piecePtr.asFunction< - int Function(ffi.Pointer, int, ffi.Pointer, int, - int, bool)>(); + late final _llama_vocab_bosPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_bos'); + late final _llama_vocab_bos = + _llama_vocab_bosPtr.asFunction)>(); - /// @details Convert the provided tokens into text (inverse of llama_tokenize()). - /// @param text The char pointer must be large enough to hold the resulting text. - /// @return Returns the number of chars/bytes on success, no more than text_len_max. - /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned. - /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. - /// @param unparse_special If true, special tokens are rendered in the output. - int llama_detokenize( + @Deprecated('use llama_vocab_bos instead') + int llama_vocab_cls( ffi.Pointer vocab, - ffi.Pointer tokens, - int n_tokens, - ffi.Pointer text, - int text_len_max, - bool remove_special, - bool unparse_special, ) { - return _llama_detokenize( + return _llama_vocab_cls( vocab, - tokens, - n_tokens, - text, - text_len_max, - remove_special, - unparse_special, ); } - late final _llama_detokenizePtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer, - ffi.Int32, - ffi.Pointer, - ffi.Int32, - ffi.Bool, - ffi.Bool)>>('llama_detokenize'); - late final _llama_detokenize = _llama_detokenizePtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, int, - ffi.Pointer, int, bool, bool)>(); + late final _llama_vocab_clsPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_cls'); + late final _llama_vocab_cls = + _llama_vocab_clsPtr.asFunction)>(); - /// Apply chat template. Inspired by hf apply_chat_template() on python. - /// - /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template - /// @param tmpl A Jinja template to use for this chat. - /// @param chat Pointer to a list of multiple llama_chat_message - /// @param n_msg Number of llama_chat_message in this chat - /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. - /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages) - /// @param length The size of the allocated buffer - /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. - int llama_chat_apply_template( - ffi.Pointer tmpl, - ffi.Pointer chat, - int n_msg, - bool add_ass, - ffi.Pointer buf, - int length, + int llama_vocab_eos( + ffi.Pointer vocab, ) { - return _llama_chat_apply_template( - tmpl, - chat, - n_msg, - add_ass, - buf, - length, + return _llama_vocab_eos( + vocab, ); } - late final _llama_chat_apply_templatePtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Bool, - ffi.Pointer, - ffi.Int32)>>('llama_chat_apply_template'); - late final _llama_chat_apply_template = - _llama_chat_apply_templatePtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer, - int, bool, ffi.Pointer, int)>(); + late final _llama_vocab_eosPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_eos'); + late final _llama_vocab_eos = + _llama_vocab_eosPtr.asFunction)>(); - int llama_chat_builtin_templates( - ffi.Pointer> output, - int len, + int llama_vocab_eot( + ffi.Pointer vocab, ) { - return _llama_chat_builtin_templates( - output, - len, + return _llama_vocab_eot( + vocab, ); } - late final _llama_chat_builtin_templatesPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function(ffi.Pointer>, - ffi.Size)>>('llama_chat_builtin_templates'); - late final _llama_chat_builtin_templates = _llama_chat_builtin_templatesPtr - .asFunction>, int)>(); + late final _llama_vocab_eotPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_eot'); + late final _llama_vocab_eot = + _llama_vocab_eotPtr.asFunction)>(); - bool llama_set_sampler( - ffi.Pointer ctx, - int seq_id, - ffi.Pointer smpl, + int llama_vocab_fim_mid( + ffi.Pointer vocab, ) { - return _llama_set_sampler( - ctx, - seq_id, - smpl, + return _llama_vocab_fim_mid( + vocab, ); } - late final _llama_set_samplerPtr = _lookup< - ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, llama_seq_id, - ffi.Pointer)>>('llama_set_sampler'); - late final _llama_set_sampler = _llama_set_samplerPtr.asFunction< - bool Function( - ffi.Pointer, int, ffi.Pointer)>(); + late final _llama_vocab_fim_midPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_fim_mid'); + late final _llama_vocab_fim_mid = _llama_vocab_fim_midPtr + .asFunction)>(); - ffi.Pointer llama_sampler_init( - ffi.Pointer iface, - llama_sampler_context_t ctx, + int llama_vocab_fim_pad( + ffi.Pointer vocab, ) { - return _llama_sampler_init( - iface, - ctx, + return _llama_vocab_fim_pad( + vocab, ); } - late final _llama_sampler_initPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - llama_sampler_context_t)>>('llama_sampler_init'); - late final _llama_sampler_init = _llama_sampler_initPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, llama_sampler_context_t)>(); + late final _llama_vocab_fim_padPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_fim_pad'); + late final _llama_vocab_fim_pad = _llama_vocab_fim_padPtr + .asFunction)>(); - ffi.Pointer llama_sampler_name( - ffi.Pointer smpl, + int llama_vocab_fim_pre( + ffi.Pointer vocab, ) { - return _llama_sampler_name( - smpl, + return _llama_vocab_fim_pre( + vocab, ); } - late final _llama_sampler_namePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('llama_sampler_name'); - late final _llama_sampler_name = _llama_sampler_namePtr - .asFunction Function(ffi.Pointer)>(); + late final _llama_vocab_fim_prePtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_fim_pre'); + late final _llama_vocab_fim_pre = _llama_vocab_fim_prePtr + .asFunction)>(); - void llama_sampler_accept( - ffi.Pointer smpl, - int token, + int llama_vocab_fim_rep( + ffi.Pointer vocab, ) { - return _llama_sampler_accept( - smpl, - token, + return _llama_vocab_fim_rep( + vocab, ); } - late final _llama_sampler_acceptPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - llama_token)>>('llama_sampler_accept'); - late final _llama_sampler_accept = _llama_sampler_acceptPtr - .asFunction, int)>(); - - void llama_sampler_apply( - ffi.Pointer smpl, - ffi.Pointer cur_p, + late final _llama_vocab_fim_repPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_fim_rep'); + late final _llama_vocab_fim_rep = _llama_vocab_fim_repPtr + .asFunction)>(); + + int llama_vocab_fim_sep( + ffi.Pointer vocab, ) { - return _llama_sampler_apply( - smpl, - cur_p, + return _llama_vocab_fim_sep( + vocab, ); } - late final _llama_sampler_applyPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('llama_sampler_apply'); - late final _llama_sampler_apply = _llama_sampler_applyPtr.asFunction< - void Function( - ffi.Pointer, ffi.Pointer)>(); + late final _llama_vocab_fim_sepPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_fim_sep'); + late final _llama_vocab_fim_sep = _llama_vocab_fim_sepPtr + .asFunction)>(); - void llama_sampler_reset( - ffi.Pointer smpl, + int llama_vocab_fim_suf( + ffi.Pointer vocab, ) { - return _llama_sampler_reset( - smpl, + return _llama_vocab_fim_suf( + vocab, ); } - late final _llama_sampler_resetPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_sampler_reset'); - late final _llama_sampler_reset = _llama_sampler_resetPtr - .asFunction)>(); + late final _llama_vocab_fim_sufPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_fim_suf'); + late final _llama_vocab_fim_suf = _llama_vocab_fim_sufPtr + .asFunction)>(); - ffi.Pointer llama_sampler_clone( - ffi.Pointer smpl, + bool llama_vocab_get_add_bos( + ffi.Pointer vocab, ) { - return _llama_sampler_clone( - smpl, + return _llama_vocab_get_add_bos( + vocab, ); } - late final _llama_sampler_clonePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('llama_sampler_clone'); - late final _llama_sampler_clone = _llama_sampler_clonePtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + late final _llama_vocab_get_add_bosPtr = + _lookup)>>( + 'llama_vocab_get_add_bos'); + late final _llama_vocab_get_add_bos = _llama_vocab_get_add_bosPtr + .asFunction)>(); - void llama_sampler_free( - ffi.Pointer smpl, + bool llama_vocab_get_add_eos( + ffi.Pointer vocab, ) { - return _llama_sampler_free( - smpl, + return _llama_vocab_get_add_eos( + vocab, ); } - late final _llama_sampler_freePtr = _lookup< - ffi.NativeFunction)>>( - 'llama_sampler_free'); - late final _llama_sampler_free = _llama_sampler_freePtr - .asFunction)>(); + late final _llama_vocab_get_add_eosPtr = + _lookup)>>( + 'llama_vocab_get_add_eos'); + late final _llama_vocab_get_add_eos = _llama_vocab_get_add_eosPtr + .asFunction)>(); - ffi.Pointer llama_sampler_chain_init( - llama_sampler_chain_params params, + bool llama_vocab_get_add_sep( + ffi.Pointer vocab, ) { - return _llama_sampler_chain_init( - params, + return _llama_vocab_get_add_sep( + vocab, ); } - late final _llama_sampler_chain_initPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - llama_sampler_chain_params)>>('llama_sampler_chain_init'); - late final _llama_sampler_chain_init = - _llama_sampler_chain_initPtr.asFunction< - ffi.Pointer Function(llama_sampler_chain_params)>(); + late final _llama_vocab_get_add_sepPtr = + _lookup)>>( + 'llama_vocab_get_add_sep'); + late final _llama_vocab_get_add_sep = _llama_vocab_get_add_sepPtr + .asFunction)>(); - void llama_sampler_chain_add( - ffi.Pointer chain, - ffi.Pointer smpl, + llama_token_attr llama_vocab_get_attr( + ffi.Pointer vocab, + Dartllama_token token, ) { - return _llama_sampler_chain_add( - chain, - smpl, - ); + return llama_token_attr.fromValue(_llama_vocab_get_attr( + vocab, + token, + )); } - late final _llama_sampler_chain_addPtr = _lookup< + late final _llama_vocab_get_attrPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('llama_sampler_chain_add'); - late final _llama_sampler_chain_add = _llama_sampler_chain_addPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + ffi.UnsignedInt Function( + ffi.Pointer, llama_token)>>('llama_vocab_get_attr'); + late final _llama_vocab_get_attr = _llama_vocab_get_attrPtr + .asFunction, int)>(); - ffi.Pointer llama_sampler_chain_get( - ffi.Pointer chain, - int i, + double llama_vocab_get_score( + ffi.Pointer vocab, + int token, ) { - return _llama_sampler_chain_get( - chain, - i, + return _llama_vocab_get_score( + vocab, + token, ); } - late final _llama_sampler_chain_getPtr = _lookup< + late final _llama_vocab_get_scorePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Int32)>>('llama_sampler_chain_get'); - late final _llama_sampler_chain_get = _llama_sampler_chain_getPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int)>(); + ffi.Float Function( + ffi.Pointer, llama_token)>>('llama_vocab_get_score'); + late final _llama_vocab_get_score = _llama_vocab_get_scorePtr + .asFunction, int)>(); - int llama_sampler_chain_n( - ffi.Pointer chain, + ffi.Pointer llama_vocab_get_suppress_tokens( + ffi.Pointer vocab, + ffi.Pointer n_suppress_tokens, ) { - return _llama_sampler_chain_n( - chain, + return _llama_vocab_get_suppress_tokens( + vocab, + n_suppress_tokens, ); } - late final _llama_sampler_chain_nPtr = - _lookup)>>( - 'llama_sampler_chain_n'); - late final _llama_sampler_chain_n = _llama_sampler_chain_nPtr - .asFunction)>(); + late final _llama_vocab_get_suppress_tokensPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('llama_vocab_get_suppress_tokens'); + late final _llama_vocab_get_suppress_tokens = + _llama_vocab_get_suppress_tokensPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer llama_sampler_chain_remove( - ffi.Pointer chain, - int i, + ffi.Pointer llama_vocab_get_text( + ffi.Pointer vocab, + int token, ) { - return _llama_sampler_chain_remove( - chain, - i, + return _llama_vocab_get_text( + vocab, + token, ); } - late final _llama_sampler_chain_removePtr = _lookup< + late final _llama_vocab_get_textPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Int32)>>('llama_sampler_chain_remove'); - late final _llama_sampler_chain_remove = - _llama_sampler_chain_removePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int)>(); - - ffi.Pointer llama_sampler_init_greedy() { - return _llama_sampler_init_greedy(); - } - - late final _llama_sampler_init_greedyPtr = - _lookup Function()>>( - 'llama_sampler_init_greedy'); - late final _llama_sampler_init_greedy = _llama_sampler_init_greedyPtr - .asFunction Function()>(); + ffi.Pointer Function( + ffi.Pointer, llama_token)>>('llama_vocab_get_text'); + late final _llama_vocab_get_text = _llama_vocab_get_textPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, int)>(); - /// seed == LLAMA_DEFAULT_SEED to use a random seed. - ffi.Pointer llama_sampler_init_dist( - int seed, + bool llama_vocab_is_control( + ffi.Pointer vocab, + int token, ) { - return _llama_sampler_init_dist( - seed, + return _llama_vocab_is_control( + vocab, + token, ); } - late final _llama_sampler_init_distPtr = _lookup< - ffi.NativeFunction Function(ffi.Uint32)>>( - 'llama_sampler_init_dist'); - late final _llama_sampler_init_dist = _llama_sampler_init_distPtr - .asFunction Function(int)>(); + late final _llama_vocab_is_controlPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer, + llama_token)>>('llama_vocab_is_control'); + late final _llama_vocab_is_control = _llama_vocab_is_controlPtr + .asFunction, int)>(); - /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 - /// Setting k <= 0 makes this a noop - ffi.Pointer llama_sampler_init_top_k( - int k, + bool llama_vocab_is_eog( + ffi.Pointer vocab, + int token, ) { - return _llama_sampler_init_top_k( - k, + return _llama_vocab_is_eog( + vocab, + token, ); } - late final _llama_sampler_init_top_kPtr = _lookup< - ffi.NativeFunction Function(ffi.Int32)>>( - 'llama_sampler_init_top_k'); - late final _llama_sampler_init_top_k = _llama_sampler_init_top_kPtr - .asFunction Function(int)>(); + late final _llama_vocab_is_eogPtr = _lookup< + ffi.NativeFunction< + ffi.Bool Function( + ffi.Pointer, llama_token)>>('llama_vocab_is_eog'); + late final _llama_vocab_is_eog = _llama_vocab_is_eogPtr + .asFunction, int)>(); - /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 - ffi.Pointer llama_sampler_init_top_p( - double p, - int min_keep, + int llama_vocab_mask( + ffi.Pointer vocab, ) { - return _llama_sampler_init_top_p( - p, - min_keep, + return _llama_vocab_mask( + vocab, ); } - late final _llama_sampler_init_top_pPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Float, ffi.Size)>>('llama_sampler_init_top_p'); - late final _llama_sampler_init_top_p = _llama_sampler_init_top_pPtr - .asFunction Function(double, int)>(); + late final _llama_vocab_maskPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_mask'); + late final _llama_vocab_mask = + _llama_vocab_maskPtr.asFunction)>(); - /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 - ffi.Pointer llama_sampler_init_min_p( - double p, - int min_keep, + int llama_vocab_n_tokens( + ffi.Pointer vocab, ) { - return _llama_sampler_init_min_p( - p, - min_keep, + return _llama_vocab_n_tokens( + vocab, ); } - late final _llama_sampler_init_min_pPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Float, ffi.Size)>>('llama_sampler_init_min_p'); - late final _llama_sampler_init_min_p = _llama_sampler_init_min_pPtr - .asFunction Function(double, int)>(); + late final _llama_vocab_n_tokensPtr = + _lookup)>>( + 'llama_vocab_n_tokens'); + late final _llama_vocab_n_tokens = _llama_vocab_n_tokensPtr + .asFunction)>(); - /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. - ffi.Pointer llama_sampler_init_typical( - double p, - int min_keep, + int llama_vocab_nl( + ffi.Pointer vocab, ) { - return _llama_sampler_init_typical( - p, - min_keep, + return _llama_vocab_nl( + vocab, ); } - late final _llama_sampler_init_typicalPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Float, ffi.Size)>>('llama_sampler_init_typical'); - late final _llama_sampler_init_typical = _llama_sampler_init_typicalPtr - .asFunction Function(double, int)>(); + late final _llama_vocab_nlPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_nl'); + late final _llama_vocab_nl = + _llama_vocab_nlPtr.asFunction)>(); - /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf - ffi.Pointer llama_sampler_init_temp( - double t, + int llama_vocab_pad( + ffi.Pointer vocab, ) { - return _llama_sampler_init_temp( - t, + return _llama_vocab_pad( + vocab, ); } - late final _llama_sampler_init_tempPtr = _lookup< - ffi.NativeFunction Function(ffi.Float)>>( - 'llama_sampler_init_temp'); - late final _llama_sampler_init_temp = _llama_sampler_init_tempPtr - .asFunction Function(double)>(); + late final _llama_vocab_padPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_pad'); + late final _llama_vocab_pad = + _llama_vocab_padPtr.asFunction)>(); - /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. - ffi.Pointer llama_sampler_init_temp_ext( - double t, - double delta, - double exponent, + int llama_vocab_sep( + ffi.Pointer vocab, ) { - return _llama_sampler_init_temp_ext( - t, - delta, - exponent, + return _llama_vocab_sep( + vocab, ); } - late final _llama_sampler_init_temp_extPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Float, ffi.Float, ffi.Float)>>('llama_sampler_init_temp_ext'); - late final _llama_sampler_init_temp_ext = - _llama_sampler_init_temp_extPtr.asFunction< - ffi.Pointer Function(double, double, double)>(); + late final _llama_vocab_sepPtr = _lookup< + ffi.NativeFunction)>>( + 'llama_vocab_sep'); + late final _llama_vocab_sep = + _llama_vocab_sepPtr.asFunction)>(); - /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 - ffi.Pointer llama_sampler_init_xtc( - double p, - double t, - int min_keep, - int seed, + llama_vocab_type llama_vocab_type$1( + ffi.Pointer vocab, ) { - return _llama_sampler_init_xtc( - p, - t, - min_keep, - seed, + return llama_vocab_type.fromValue(_llama_vocab_type( + vocab, + )); + } + + late final _llama_vocab_type$1Ptr = _lookup< + ffi + .NativeFunction)>>( + 'llama_vocab_type'); + late final _llama_vocab_type = _llama_vocab_type$1Ptr + .asFunction)>(); + + int mtmd_batch_add_chunk( + ffi.Pointer batch, + ffi.Pointer chunk, + ) { + return _mtmd_batch_add_chunk( + batch, + chunk, ); } - late final _llama_sampler_init_xtcPtr = _lookup< + late final _mtmd_batch_add_chunkPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Float, ffi.Float, ffi.Size, - ffi.Uint32)>>('llama_sampler_init_xtc'); - late final _llama_sampler_init_xtc = _llama_sampler_init_xtcPtr.asFunction< - ffi.Pointer Function(double, double, int, int)>(); + ffi.Int32 Function(ffi.Pointer, + ffi.Pointer)>>('mtmd_batch_add_chunk'); + late final _mtmd_batch_add_chunk = _mtmd_batch_add_chunkPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 - ffi.Pointer llama_sampler_init_top_n_sigma( - double n, + int mtmd_batch_encode( + ffi.Pointer batch, ) { - return _llama_sampler_init_top_n_sigma( - n, + return _mtmd_batch_encode( + batch, ); } - late final _llama_sampler_init_top_n_sigmaPtr = _lookup< - ffi.NativeFunction Function(ffi.Float)>>( - 'llama_sampler_init_top_n_sigma'); - late final _llama_sampler_init_top_n_sigma = - _llama_sampler_init_top_n_sigmaPtr - .asFunction Function(double)>(); + late final _mtmd_batch_encodePtr = + _lookup)>>( + 'mtmd_batch_encode'); + late final _mtmd_batch_encode = + _mtmd_batch_encodePtr.asFunction)>(); - /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. - /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. - /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. - /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. - ffi.Pointer llama_sampler_init_mirostat( - int n_vocab, - int seed, - double tau, - double eta, - int m, + void mtmd_batch_free( + ffi.Pointer batch, ) { - return _llama_sampler_init_mirostat( - n_vocab, - seed, - tau, - eta, - m, + return _mtmd_batch_free( + batch, ); } - late final _llama_sampler_init_mirostatPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Int32, ffi.Uint32, ffi.Float, - ffi.Float, ffi.Int32)>>('llama_sampler_init_mirostat'); - late final _llama_sampler_init_mirostat = - _llama_sampler_init_mirostatPtr.asFunction< - ffi.Pointer Function(int, int, double, double, int)>(); + late final _mtmd_batch_freePtr = + _lookup)>>( + 'mtmd_batch_free'); + late final _mtmd_batch_free = + _mtmd_batch_freePtr.asFunction)>(); - /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. - /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. - /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. - ffi.Pointer llama_sampler_init_mirostat_v2( - int seed, - double tau, - double eta, + ffi.Pointer mtmd_batch_get_output_embd( + ffi.Pointer batch, + ffi.Pointer chunk, ) { - return _llama_sampler_init_mirostat_v2( - seed, - tau, - eta, + return _mtmd_batch_get_output_embd( + batch, + chunk, ); } - late final _llama_sampler_init_mirostat_v2Ptr = _lookup< + late final _mtmd_batch_get_output_embdPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Uint32, ffi.Float, - ffi.Float)>>('llama_sampler_init_mirostat_v2'); - late final _llama_sampler_init_mirostat_v2 = - _llama_sampler_init_mirostat_v2Ptr.asFunction< - ffi.Pointer Function(int, double, double)>(); + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('mtmd_batch_get_output_embd'); + late final _mtmd_batch_get_output_embd = + _mtmd_batch_get_output_embdPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - /// @details Initializes a GBNF grammar, see grammars/README.md for details. - /// @param vocab The vocabulary that this grammar will be used with. - /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. - /// @param grammar_root The name of the start symbol for the grammar. - ffi.Pointer llama_sampler_init_grammar( - ffi.Pointer vocab, - ffi.Pointer grammar_str, - ffi.Pointer grammar_root, + ffi.Pointer mtmd_batch_init( + ffi.Pointer ctx, ) { - return _llama_sampler_init_grammar( - vocab, - grammar_str, - grammar_root, + return _mtmd_batch_init( + ctx, ); } - late final _llama_sampler_init_grammarPtr = _lookup< + late final _mtmd_batch_initPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer)>>('llama_sampler_init_grammar'); - late final _llama_sampler_init_grammar = - _llama_sampler_init_grammarPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer)>>('mtmd_batch_init'); + late final _mtmd_batch_init = _mtmd_batch_initPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - ffi.Pointer llama_sampler_init_grammar_lazy( - ffi.Pointer vocab, - ffi.Pointer grammar_str, - ffi.Pointer grammar_root, - ffi.Pointer> trigger_words, - int num_trigger_words, - ffi.Pointer trigger_tokens, - int num_trigger_tokens, + void mtmd_bitmap_free( + ffi.Pointer bitmap, ) { - return _llama_sampler_init_grammar_lazy( - vocab, - grammar_str, - grammar_root, - trigger_words, - num_trigger_words, - trigger_tokens, - num_trigger_tokens, + return _mtmd_bitmap_free( + bitmap, ); } - late final _llama_sampler_init_grammar_lazyPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer>, - ffi.Size, - ffi.Pointer, - ffi.Size)>>('llama_sampler_init_grammar_lazy'); - late final _llama_sampler_init_grammar_lazy = - _llama_sampler_init_grammar_lazyPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer>, - int, - ffi.Pointer, - int)>(); + late final _mtmd_bitmap_freePtr = + _lookup)>>( + 'mtmd_bitmap_free'); + late final _mtmd_bitmap_free = _mtmd_bitmap_freePtr + .asFunction)>(); - /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 - /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. - /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. - ffi.Pointer llama_sampler_init_grammar_lazy_patterns( - ffi.Pointer vocab, - ffi.Pointer grammar_str, - ffi.Pointer grammar_root, - ffi.Pointer> trigger_patterns, - int num_trigger_patterns, - ffi.Pointer trigger_tokens, - int num_trigger_tokens, + ffi.Pointer mtmd_bitmap_get_data( + ffi.Pointer bitmap, ) { - return _llama_sampler_init_grammar_lazy_patterns( - vocab, - grammar_str, - grammar_root, - trigger_patterns, - num_trigger_patterns, - trigger_tokens, - num_trigger_tokens, + return _mtmd_bitmap_get_data( + bitmap, ); } - late final _llama_sampler_init_grammar_lazy_patternsPtr = _lookup< + late final _mtmd_bitmap_get_dataPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer>, - ffi.Size, - ffi.Pointer, - ffi.Size)>>('llama_sampler_init_grammar_lazy_patterns'); - late final _llama_sampler_init_grammar_lazy_patterns = - _llama_sampler_init_grammar_lazy_patternsPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer>, - int, - ffi.Pointer, - int)>(); + ffi.Pointer Function( + ffi.Pointer)>>('mtmd_bitmap_get_data'); + late final _mtmd_bitmap_get_data = _mtmd_bitmap_get_dataPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. - ffi.Pointer llama_sampler_init_penalties( - int penalty_last_n, - double penalty_repeat, - double penalty_freq, - double penalty_present, + ffi.Pointer mtmd_bitmap_get_id( + ffi.Pointer bitmap, ) { - return _llama_sampler_init_penalties( - penalty_last_n, - penalty_repeat, - penalty_freq, - penalty_present, + return _mtmd_bitmap_get_id( + bitmap, ); } - late final _llama_sampler_init_penaltiesPtr = _lookup< + late final _mtmd_bitmap_get_idPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Int32, ffi.Float, ffi.Float, - ffi.Float)>>('llama_sampler_init_penalties'); - late final _llama_sampler_init_penalties = - _llama_sampler_init_penaltiesPtr.asFunction< - ffi.Pointer Function(int, double, double, double)>(); + ffi.Pointer Function( + ffi.Pointer)>>('mtmd_bitmap_get_id'); + late final _mtmd_bitmap_get_id = _mtmd_bitmap_get_idPtr + .asFunction Function(ffi.Pointer)>(); - /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 - ffi.Pointer llama_sampler_init_dry( - ffi.Pointer vocab, - int n_ctx_train, - double dry_multiplier, - double dry_base, - int dry_allowed_length, - int dry_penalty_last_n, - ffi.Pointer> seq_breakers, - int num_breakers, + int mtmd_bitmap_get_n_bytes( + ffi.Pointer bitmap, ) { - return _llama_sampler_init_dry( - vocab, - n_ctx_train, - dry_multiplier, - dry_base, - dry_allowed_length, - dry_penalty_last_n, - seq_breakers, - num_breakers, + return _mtmd_bitmap_get_n_bytes( + bitmap, ); } - late final _llama_sampler_init_dryPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Int32, - ffi.Float, - ffi.Float, - ffi.Int32, - ffi.Int32, - ffi.Pointer>, - ffi.Size)>>('llama_sampler_init_dry'); - late final _llama_sampler_init_dry = _llama_sampler_init_dryPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, int, double, - double, int, int, ffi.Pointer>, int)>(); + late final _mtmd_bitmap_get_n_bytesPtr = + _lookup)>>( + 'mtmd_bitmap_get_n_bytes'); + late final _mtmd_bitmap_get_n_bytes = _mtmd_bitmap_get_n_bytesPtr + .asFunction)>(); - /// adaptive-p: select tokens near a configurable target probability over time. - /// - /// the adaptive-p sampler transforms the token probability distribution to favor tokens - /// that fall near a user-configurable probability target. - /// - /// internally, the sampler maintains an exponential moving average of the *ORIGINAL* - /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an - /// adapted target probability at each sampling step, thus maintaining the desired target - /// probability over time. - /// - /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last - /// in the sampler chain (like mirostat, dist, greedy). - /// - /// only mild truncation before this sampler is recommended. we suggest applying min-p - /// before adaptive-p as the only other active sampler in the chain. - /// - /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) - /// @param decay EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99) - /// @param seed RNG seed - /// - /// ref: https://github.com/ggml-org/llama.cpp/pull/17927 - ffi.Pointer llama_sampler_init_adaptive_p( - double target, - double decay, - int seed, + int mtmd_bitmap_get_nx( + ffi.Pointer bitmap, ) { - return _llama_sampler_init_adaptive_p( - target, - decay, - seed, + return _mtmd_bitmap_get_nx( + bitmap, ); } - late final _llama_sampler_init_adaptive_pPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Float, ffi.Float, - ffi.Uint32)>>('llama_sampler_init_adaptive_p'); - late final _llama_sampler_init_adaptive_p = _llama_sampler_init_adaptive_pPtr - .asFunction Function(double, double, int)>(); + late final _mtmd_bitmap_get_nxPtr = _lookup< + ffi.NativeFunction)>>( + 'mtmd_bitmap_get_nx'); + late final _mtmd_bitmap_get_nx = _mtmd_bitmap_get_nxPtr + .asFunction)>(); - ffi.Pointer llama_sampler_init_logit_bias( - int n_vocab, - int n_logit_bias, - ffi.Pointer logit_bias, + int mtmd_bitmap_get_ny( + ffi.Pointer bitmap, ) { - return _llama_sampler_init_logit_bias( - n_vocab, - n_logit_bias, - logit_bias, + return _mtmd_bitmap_get_ny( + bitmap, ); } - late final _llama_sampler_init_logit_biasPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Int32, ffi.Int32, - ffi.Pointer)>>('llama_sampler_init_logit_bias'); - late final _llama_sampler_init_logit_bias = - _llama_sampler_init_logit_biasPtr.asFunction< - ffi.Pointer Function( - int, int, ffi.Pointer)>(); + late final _mtmd_bitmap_get_nyPtr = _lookup< + ffi.NativeFunction)>>( + 'mtmd_bitmap_get_ny'); + late final _mtmd_bitmap_get_ny = _mtmd_bitmap_get_nyPtr + .asFunction)>(); - ffi.Pointer llama_sampler_init_infill( - ffi.Pointer vocab, + ffi.Pointer mtmd_bitmap_init( + int nx, + int ny, + ffi.Pointer data, ) { - return _llama_sampler_init_infill( - vocab, + return _mtmd_bitmap_init( + nx, + ny, + data, ); } - late final _llama_sampler_init_infillPtr = _lookup< + late final _mtmd_bitmap_initPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('llama_sampler_init_infill'); - late final _llama_sampler_init_infill = - _llama_sampler_init_infillPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + ffi.Pointer Function(ffi.Uint32, ffi.Uint32, + ffi.Pointer)>>('mtmd_bitmap_init'); + late final _mtmd_bitmap_init = _mtmd_bitmap_initPtr.asFunction< + ffi.Pointer Function( + int, int, ffi.Pointer)>(); - int llama_sampler_get_seed( - ffi.Pointer smpl, + ffi.Pointer mtmd_bitmap_init_from_audio( + int n_samples, + ffi.Pointer data, ) { - return _llama_sampler_get_seed( - smpl, + return _mtmd_bitmap_init_from_audio( + n_samples, + data, ); } - late final _llama_sampler_get_seedPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_sampler_get_seed'); - late final _llama_sampler_get_seed = _llama_sampler_get_seedPtr - .asFunction)>(); + late final _mtmd_bitmap_init_from_audioPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Size, + ffi.Pointer)>>('mtmd_bitmap_init_from_audio'); + late final _mtmd_bitmap_init_from_audio = + _mtmd_bitmap_init_from_audioPtr.asFunction< + ffi.Pointer Function(int, ffi.Pointer)>(); - int llama_sampler_sample( - ffi.Pointer smpl, - ffi.Pointer ctx, - int idx, + ffi.Pointer mtmd_bitmap_init_lazy( + ffi.Pointer ctx, + ffi.Pointer id, + ffi.Pointer user_data, + mtmd_bitmap_lazy_callback callback, ) { - return _llama_sampler_sample( - smpl, + return _mtmd_bitmap_init_lazy( ctx, - idx, + id, + user_data, + callback, ); } - late final _llama_sampler_samplePtr = _lookup< + late final _mtmd_bitmap_init_lazyPtr = _lookup< ffi.NativeFunction< - llama_token Function(ffi.Pointer, - ffi.Pointer, ffi.Int32)>>('llama_sampler_sample'); - late final _llama_sampler_sample = _llama_sampler_samplePtr.asFunction< - int Function( - ffi.Pointer, ffi.Pointer, int)>(); - - /// @details Build a split GGUF final path for this chunk. - /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" - int llama_split_path( - ffi.Pointer split_path, - int maxlen, - ffi.Pointer path_prefix, - int split_no, - int split_count, + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + mtmd_bitmap_lazy_callback)>>('mtmd_bitmap_init_lazy'); + late final _mtmd_bitmap_init_lazy = _mtmd_bitmap_init_lazyPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + mtmd_bitmap_lazy_callback)>(); + + bool mtmd_bitmap_is_audio( + ffi.Pointer bitmap, ) { - return _llama_split_path( - split_path, - maxlen, - path_prefix, - split_no, - split_count, + return _mtmd_bitmap_is_audio( + bitmap, ); } - late final _llama_split_pathPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Size, - ffi.Pointer, - ffi.Int32, - ffi.Int32)>>('llama_split_path'); - late final _llama_split_path = _llama_split_pathPtr.asFunction< - int Function( - ffi.Pointer, int, ffi.Pointer, int, int)>(); + late final _mtmd_bitmap_is_audioPtr = + _lookup)>>( + 'mtmd_bitmap_is_audio'); + late final _mtmd_bitmap_is_audio = _mtmd_bitmap_is_audioPtr + .asFunction)>(); - /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. - /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" - int llama_split_prefix( - ffi.Pointer split_prefix, - int maxlen, - ffi.Pointer split_path, - int split_no, - int split_count, + void mtmd_bitmap_set_id( + ffi.Pointer bitmap, + ffi.Pointer id, ) { - return _llama_split_prefix( - split_prefix, - maxlen, - split_path, - split_no, - split_count, + return _mtmd_bitmap_set_id( + bitmap, + id, ); } - late final _llama_split_prefixPtr = _lookup< + late final _mtmd_bitmap_set_idPtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Size, - ffi.Pointer, - ffi.Int32, - ffi.Int32)>>('llama_split_prefix'); - late final _llama_split_prefix = _llama_split_prefixPtr.asFunction< - int Function( - ffi.Pointer, int, ffi.Pointer, int, int)>(); + ffi.Void Function(ffi.Pointer, + ffi.Pointer)>>('mtmd_bitmap_set_id'); + late final _mtmd_bitmap_set_id = _mtmd_bitmap_set_idPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer llama_print_system_info() { - return _llama_print_system_info(); + mtmd_context_params mtmd_context_params_default() { + return _mtmd_context_params_default(); } - late final _llama_print_system_infoPtr = - _lookup Function()>>( - 'llama_print_system_info'); - late final _llama_print_system_info = _llama_print_system_infoPtr - .asFunction Function()>(); + late final _mtmd_context_params_defaultPtr = + _lookup>( + 'mtmd_context_params_default'); + late final _mtmd_context_params_default = _mtmd_context_params_defaultPtr + .asFunction(); - void llama_log_get( - ffi.Pointer log_callback, - ffi.Pointer> user_data, + bool mtmd_decode_use_mrope( + ffi.Pointer ctx, ) { - return _llama_log_get( - log_callback, - user_data, + return _mtmd_decode_use_mrope( + ctx, ); } - late final _llama_log_getPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer>)>>('llama_log_get'); - late final _llama_log_get = _llama_log_getPtr.asFunction< - void Function(ffi.Pointer, - ffi.Pointer>)>(); + late final _mtmd_decode_use_mropePtr = + _lookup)>>( + 'mtmd_decode_use_mrope'); + late final _mtmd_decode_use_mrope = _mtmd_decode_use_mropePtr + .asFunction)>(); - void llama_log_set( - ggml_log_callback log_callback, - ffi.Pointer user_data, + bool mtmd_decode_use_non_causal( + ffi.Pointer ctx, + ffi.Pointer chunk, ) { - return _llama_log_set( - log_callback, - user_data, + return _mtmd_decode_use_non_causal( + ctx, + chunk, ); } - late final _llama_log_setPtr = _lookup< + late final _mtmd_decode_use_non_causalPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_log_callback, ffi.Pointer)>>('llama_log_set'); - late final _llama_log_set = _llama_log_setPtr - .asFunction)>(); + ffi.Bool Function(ffi.Pointer, + ffi.Pointer)>>('mtmd_decode_use_non_causal'); + late final _mtmd_decode_use_non_causal = + _mtmd_decode_use_non_causalPtr.asFunction< + bool Function( + ffi.Pointer, ffi.Pointer)>(); - llama_perf_context_data llama_perf_context( - ffi.Pointer ctx, + ffi.Pointer mtmd_default_marker() { + return _mtmd_default_marker(); + } + + late final _mtmd_default_markerPtr = + _lookup Function()>>( + 'mtmd_default_marker'); + late final _mtmd_default_marker = + _mtmd_default_markerPtr.asFunction Function()>(); + + @Deprecated('use mtmd_encode_chunk() instead') + int mtmd_encode( + ffi.Pointer ctx, + ffi.Pointer image_tokens, ) { - return _llama_perf_context( + return _mtmd_encode( ctx, + image_tokens, ); } - late final _llama_perf_contextPtr = _lookup< + late final _mtmd_encodePtr = _lookup< ffi.NativeFunction< - llama_perf_context_data Function( - ffi.Pointer)>>('llama_perf_context'); - late final _llama_perf_context = _llama_perf_contextPtr.asFunction< - llama_perf_context_data Function(ffi.Pointer)>(); + ffi.Int32 Function(ffi.Pointer, + ffi.Pointer)>>('mtmd_encode'); + late final _mtmd_encode = _mtmd_encodePtr.asFunction< + int Function( + ffi.Pointer, ffi.Pointer)>(); - void llama_perf_context_print( - ffi.Pointer ctx, + int mtmd_encode_chunk( + ffi.Pointer ctx, + ffi.Pointer chunk, ) { - return _llama_perf_context_print( + return _mtmd_encode_chunk( ctx, + chunk, ); } - late final _llama_perf_context_printPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_perf_context_print'); - late final _llama_perf_context_print = _llama_perf_context_printPtr - .asFunction)>(); + late final _mtmd_encode_chunkPtr = _lookup< + ffi.NativeFunction< + ffi.Int32 Function(ffi.Pointer, + ffi.Pointer)>>('mtmd_encode_chunk'); + late final _mtmd_encode_chunk = _mtmd_encode_chunkPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer)>(); - void llama_perf_context_reset( - ffi.Pointer ctx, + void mtmd_free( + ffi.Pointer ctx, ) { - return _llama_perf_context_reset( + return _mtmd_free( ctx, ); } - late final _llama_perf_context_resetPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_perf_context_reset'); - late final _llama_perf_context_reset = _llama_perf_context_resetPtr - .asFunction)>(); + late final _mtmd_freePtr = + _lookup)>>( + 'mtmd_free'); + late final _mtmd_free = + _mtmd_freePtr.asFunction)>(); - llama_perf_sampler_data llama_perf_sampler( - ffi.Pointer chain, + int mtmd_get_audio_sample_rate( + ffi.Pointer ctx, ) { - return _llama_perf_sampler( - chain, + return _mtmd_get_audio_sample_rate( + ctx, ); } - late final _llama_perf_samplerPtr = _lookup< - ffi.NativeFunction< - llama_perf_sampler_data Function( - ffi.Pointer)>>('llama_perf_sampler'); - late final _llama_perf_sampler = _llama_perf_samplerPtr.asFunction< - llama_perf_sampler_data Function(ffi.Pointer)>(); + late final _mtmd_get_audio_sample_ratePtr = + _lookup)>>( + 'mtmd_get_audio_sample_rate'); + late final _mtmd_get_audio_sample_rate = _mtmd_get_audio_sample_ratePtr + .asFunction)>(); - void llama_perf_sampler_print( - ffi.Pointer chain, + mtmd_caps mtmd_get_cap_from_file( + ffi.Pointer mmproj_fname, ) { - return _llama_perf_sampler_print( - chain, + return _mtmd_get_cap_from_file( + mmproj_fname, ); } - late final _llama_perf_sampler_printPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_perf_sampler_print'); - late final _llama_perf_sampler_print = _llama_perf_sampler_printPtr - .asFunction)>(); + late final _mtmd_get_cap_from_filePtr = + _lookup)>>( + 'mtmd_get_cap_from_file'); + late final _mtmd_get_cap_from_file = _mtmd_get_cap_from_filePtr + .asFunction)>(); - void llama_perf_sampler_reset( - ffi.Pointer chain, + ffi.Pointer mtmd_get_marker( + ffi.Pointer ctx, ) { - return _llama_perf_sampler_reset( - chain, + return _mtmd_get_marker( + ctx, ); } - late final _llama_perf_sampler_resetPtr = _lookup< - ffi.NativeFunction)>>( - 'llama_perf_sampler_reset'); - late final _llama_perf_sampler_reset = _llama_perf_sampler_resetPtr - .asFunction)>(); - - bool llama_opt_param_filter_all( - ffi.Pointer tensor, - ffi.Pointer userdata, - ) { - return _llama_opt_param_filter_all( - tensor, - userdata, - ); - } - - late final _llama_opt_param_filter_allPtr = _lookup< + late final _mtmd_get_markerPtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, - ffi.Pointer)>>('llama_opt_param_filter_all'); - late final _llama_opt_param_filter_all = - _llama_opt_param_filter_allPtr.asFunction< - bool Function(ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer)>>('mtmd_get_marker'); + late final _mtmd_get_marker = _mtmd_get_markerPtr + .asFunction Function(ffi.Pointer)>(); - void llama_opt_init( - ffi.Pointer lctx, - ffi.Pointer model, - llama_opt_params lopt_params, + ffi.Pointer mtmd_get_output_embd( + ffi.Pointer ctx, ) { - return _llama_opt_init( - lctx, - model, - lopt_params, + return _mtmd_get_output_embd( + ctx, ); } - late final _llama_opt_initPtr = _lookup< + late final _mtmd_get_output_embdPtr = _lookup< ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer, llama_opt_params)>>('llama_opt_init'); - late final _llama_opt_init = _llama_opt_initPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer, - llama_opt_params)>(); + ffi.Pointer Function( + ffi.Pointer)>>('mtmd_get_output_embd'); + late final _mtmd_get_output_embd = _mtmd_get_output_embdPtr + .asFunction Function(ffi.Pointer)>(); - void llama_opt_epoch( - ffi.Pointer lctx, - ggml_opt_dataset_t dataset, - ggml_opt_result_t result_train, - ggml_opt_result_t result_eval, - int idata_split, - ggml_opt_epoch_callback callback_train, - ggml_opt_epoch_callback callback_eval, + mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf( + ffi.Pointer ctx, + ffi.Pointer buf, + int len, + bool placeholder, ) { - return _llama_opt_epoch( - lctx, - dataset, - result_train, - result_eval, - idata_split, - callback_train, - callback_eval, + return _mtmd_helper_bitmap_init_from_buf( + ctx, + buf, + len, + placeholder, ); } - late final _llama_opt_epochPtr = _lookup< + late final _mtmd_helper_bitmap_init_from_bufPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer, - ggml_opt_dataset_t, - ggml_opt_result_t, - ggml_opt_result_t, - ffi.Int64, - ggml_opt_epoch_callback, - ggml_opt_epoch_callback)>>('llama_opt_epoch'); - late final _llama_opt_epoch = _llama_opt_epochPtr.asFunction< - void Function( - ffi.Pointer, - ggml_opt_dataset_t, - ggml_opt_result_t, - ggml_opt_result_t, - int, - ggml_opt_epoch_callback, - ggml_opt_epoch_callback)>(); - - ffi.Pointer mtmd_default_marker() { - return _mtmd_default_marker(); - } - - late final _mtmd_default_markerPtr = - _lookup Function()>>( - 'mtmd_default_marker'); - late final _mtmd_default_marker = - _mtmd_default_markerPtr.asFunction Function()>(); - - mtmd_context_params mtmd_context_params_default() { - return _mtmd_context_params_default(); - } - - late final _mtmd_context_params_defaultPtr = - _lookup>( - 'mtmd_context_params_default'); - late final _mtmd_context_params_default = _mtmd_context_params_defaultPtr - .asFunction(); + mtmd_helper_bitmap_wrapper Function( + ffi.Pointer, + ffi.Pointer, + ffi.Size, + ffi.Bool)>>('mtmd_helper_bitmap_init_from_buf'); + late final _mtmd_helper_bitmap_init_from_buf = + _mtmd_helper_bitmap_init_from_bufPtr.asFunction< + mtmd_helper_bitmap_wrapper Function(ffi.Pointer, + ffi.Pointer, int, bool)>(); - ffi.Pointer mtmd_init_from_file( - ffi.Pointer mmproj_fname, - ffi.Pointer text_model, - mtmd_context_params ctx_params, + mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file( + ffi.Pointer ctx, + ffi.Pointer fname, + bool placeholder, ) { - return _mtmd_init_from_file( - mmproj_fname, - text_model, - ctx_params, + return _mtmd_helper_bitmap_init_from_file( + ctx, + fname, + placeholder, ); } - late final _mtmd_init_from_filePtr = _lookup< + late final _mtmd_helper_bitmap_init_from_filePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( + mtmd_helper_bitmap_wrapper Function( + ffi.Pointer, ffi.Pointer, - ffi.Pointer, - mtmd_context_params)>>('mtmd_init_from_file'); - late final _mtmd_init_from_file = _mtmd_init_from_filePtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, mtmd_context_params)>(); + ffi.Bool)>>('mtmd_helper_bitmap_init_from_file'); + late final _mtmd_helper_bitmap_init_from_file = + _mtmd_helper_bitmap_init_from_filePtr.asFunction< + mtmd_helper_bitmap_wrapper Function( + ffi.Pointer, ffi.Pointer, bool)>(); - void mtmd_free( + int mtmd_helper_decode_image_chunk( ffi.Pointer ctx, + ffi.Pointer lctx, + ffi.Pointer chunk, + ffi.Pointer encoded_embd, + int n_past, + int seq_id, + int n_batch, + ffi.Pointer new_n_past, + mtmd_helper_post_decode_callback callback, + ffi.Pointer user_data, ) { - return _mtmd_free( + return _mtmd_helper_decode_image_chunk( ctx, + lctx, + chunk, + encoded_embd, + n_past, + seq_id, + n_batch, + new_n_past, + callback, + user_data, ); } - late final _mtmd_freePtr = - _lookup)>>( - 'mtmd_free'); - late final _mtmd_free = - _mtmd_freePtr.asFunction)>(); + late final _mtmd_helper_decode_image_chunkPtr = _lookup< + ffi.NativeFunction< + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + llama_pos, + llama_seq_id, + ffi.Int32, + ffi.Pointer, + mtmd_helper_post_decode_callback, + ffi.Pointer)>>('mtmd_helper_decode_image_chunk'); + late final _mtmd_helper_decode_image_chunk = + _mtmd_helper_decode_image_chunkPtr.asFunction< + int Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + ffi.Pointer, + mtmd_helper_post_decode_callback, + ffi.Pointer)>(); - bool mtmd_decode_use_non_causal( + int mtmd_helper_eval_chunk_single( ffi.Pointer ctx, + ffi.Pointer lctx, ffi.Pointer chunk, + int n_past, + int seq_id, + int n_batch, + bool logits_last, + ffi.Pointer new_n_past, ) { - return _mtmd_decode_use_non_causal( + return _mtmd_helper_eval_chunk_single( ctx, + lctx, chunk, + n_past, + seq_id, + n_batch, + logits_last, + new_n_past, ); } - late final _mtmd_decode_use_non_causalPtr = _lookup< + late final _mtmd_helper_eval_chunk_singlePtr = _lookup< ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer, - ffi.Pointer)>>('mtmd_decode_use_non_causal'); - late final _mtmd_decode_use_non_causal = - _mtmd_decode_use_non_causalPtr.asFunction< - bool Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + llama_pos, + llama_seq_id, + ffi.Int32, + ffi.Bool, + ffi.Pointer)>>('mtmd_helper_eval_chunk_single'); + late final _mtmd_helper_eval_chunk_single = + _mtmd_helper_eval_chunk_singlePtr.asFunction< + int Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + bool, + ffi.Pointer)>(); - bool mtmd_decode_use_mrope( + int mtmd_helper_eval_chunks( ffi.Pointer ctx, + ffi.Pointer lctx, + ffi.Pointer chunks, + int n_past, + int seq_id, + int n_batch, + bool logits_last, + ffi.Pointer new_n_past, ) { - return _mtmd_decode_use_mrope( + return _mtmd_helper_eval_chunks( ctx, + lctx, + chunks, + n_past, + seq_id, + n_batch, + logits_last, + new_n_past, ); } - late final _mtmd_decode_use_mropePtr = - _lookup)>>( - 'mtmd_decode_use_mrope'); - late final _mtmd_decode_use_mrope = _mtmd_decode_use_mropePtr - .asFunction)>(); + late final _mtmd_helper_eval_chunksPtr = _lookup< + ffi.NativeFunction< + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + llama_pos, + llama_seq_id, + ffi.Int32, + ffi.Bool, + ffi.Pointer)>>('mtmd_helper_eval_chunks'); + late final _mtmd_helper_eval_chunks = _mtmd_helper_eval_chunksPtr.asFunction< + int Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + int, + int, + int, + bool, + ffi.Pointer)>(); - bool mtmd_support_vision( - ffi.Pointer ctx, + int mtmd_helper_get_n_pos( + ffi.Pointer chunks, ) { - return _mtmd_support_vision( - ctx, + return _mtmd_helper_get_n_pos( + chunks, ); } - late final _mtmd_support_visionPtr = - _lookup)>>( - 'mtmd_support_vision'); - late final _mtmd_support_vision = _mtmd_support_visionPtr - .asFunction)>(); + late final _mtmd_helper_get_n_posPtr = _lookup< + ffi + .NativeFunction)>>( + 'mtmd_helper_get_n_pos'); + late final _mtmd_helper_get_n_pos = _mtmd_helper_get_n_posPtr + .asFunction)>(); - bool mtmd_support_audio( - ffi.Pointer ctx, + int mtmd_helper_get_n_tokens( + ffi.Pointer chunks, ) { - return _mtmd_support_audio( - ctx, + return _mtmd_helper_get_n_tokens( + chunks, ); } - late final _mtmd_support_audioPtr = - _lookup)>>( - 'mtmd_support_audio'); - late final _mtmd_support_audio = _mtmd_support_audioPtr - .asFunction)>(); + late final _mtmd_helper_get_n_tokensPtr = _lookup< + ffi + .NativeFunction)>>( + 'mtmd_helper_get_n_tokens'); + late final _mtmd_helper_get_n_tokens = _mtmd_helper_get_n_tokensPtr + .asFunction)>(); - int mtmd_get_audio_sample_rate( - ffi.Pointer ctx, + void mtmd_helper_image_get_decoder_pos( + ffi.Pointer image, + int pos_0, + ffi.Pointer out_pos, ) { - return _mtmd_get_audio_sample_rate( - ctx, + return _mtmd_helper_image_get_decoder_pos( + image, + pos_0, + out_pos, ); } - late final _mtmd_get_audio_sample_ratePtr = - _lookup)>>( - 'mtmd_get_audio_sample_rate'); - late final _mtmd_get_audio_sample_rate = _mtmd_get_audio_sample_ratePtr - .asFunction)>(); + late final _mtmd_helper_image_get_decoder_posPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, llama_pos, + ffi.Pointer)>>( + 'mtmd_helper_image_get_decoder_pos'); + late final _mtmd_helper_image_get_decoder_pos = + _mtmd_helper_image_get_decoder_posPtr.asFunction< + void Function(ffi.Pointer, int, + ffi.Pointer)>(); - ffi.Pointer mtmd_get_marker( - ffi.Pointer ctx, + void mtmd_helper_log_set( + ggml_log_callback log_callback, + ffi.Pointer user_data, ) { - return _mtmd_get_marker( - ctx, + return _mtmd_helper_log_set( + log_callback, + user_data, ); } - late final _mtmd_get_markerPtr = _lookup< + late final _mtmd_helper_log_setPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('mtmd_get_marker'); - late final _mtmd_get_marker = _mtmd_get_markerPtr - .asFunction Function(ffi.Pointer)>(); + ffi.Void Function(ggml_log_callback, + ffi.Pointer)>>('mtmd_helper_log_set'); + late final _mtmd_helper_log_set = _mtmd_helper_log_setPtr + .asFunction)>(); - ffi.Pointer mtmd_bitmap_init( - int nx, - int ny, - ffi.Pointer data, + bool mtmd_helper_support_video( + ffi.Pointer ctx, ) { - return _mtmd_bitmap_init( - nx, - ny, - data, + return _mtmd_helper_support_video( + ctx, ); } - late final _mtmd_bitmap_initPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Uint32, ffi.Uint32, - ffi.Pointer)>>('mtmd_bitmap_init'); - late final _mtmd_bitmap_init = _mtmd_bitmap_initPtr.asFunction< - ffi.Pointer Function( - int, int, ffi.Pointer)>(); + late final _mtmd_helper_support_videoPtr = + _lookup)>>( + 'mtmd_helper_support_video'); + late final _mtmd_helper_support_video = _mtmd_helper_support_videoPtr + .asFunction)>(); - ffi.Pointer mtmd_bitmap_init_from_audio( - int n_samples, - ffi.Pointer data, + void mtmd_helper_video_free( + ffi.Pointer ctx, ) { - return _mtmd_bitmap_init_from_audio( - n_samples, - data, + return _mtmd_helper_video_free( + ctx, ); } - late final _mtmd_bitmap_init_from_audioPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Size, - ffi.Pointer)>>('mtmd_bitmap_init_from_audio'); - late final _mtmd_bitmap_init_from_audio = - _mtmd_bitmap_init_from_audioPtr.asFunction< - ffi.Pointer Function(int, ffi.Pointer)>(); + late final _mtmd_helper_video_freePtr = _lookup< + ffi + .NativeFunction)>>( + 'mtmd_helper_video_free'); + late final _mtmd_helper_video_free = _mtmd_helper_video_freePtr + .asFunction)>(); - int mtmd_bitmap_get_nx( - ffi.Pointer bitmap, + mtmd_helper_video_info mtmd_helper_video_get_info( + ffi.Pointer ctx, ) { - return _mtmd_bitmap_get_nx( - bitmap, + return _mtmd_helper_video_get_info( + ctx, ); } - late final _mtmd_bitmap_get_nxPtr = _lookup< - ffi.NativeFunction)>>( - 'mtmd_bitmap_get_nx'); - late final _mtmd_bitmap_get_nx = _mtmd_bitmap_get_nxPtr - .asFunction)>(); + late final _mtmd_helper_video_get_infoPtr = _lookup< + ffi.NativeFunction< + mtmd_helper_video_info Function( + ffi.Pointer)>>('mtmd_helper_video_get_info'); + late final _mtmd_helper_video_get_info = + _mtmd_helper_video_get_infoPtr.asFunction< + mtmd_helper_video_info Function(ffi.Pointer)>(); - int mtmd_bitmap_get_ny( - ffi.Pointer bitmap, + ffi.Pointer mtmd_helper_video_init( + ffi.Pointer mctx, + ffi.Pointer path, + mtmd_helper_video_init_params params, ) { - return _mtmd_bitmap_get_ny( - bitmap, + return _mtmd_helper_video_init( + mctx, + path, + params, ); } - late final _mtmd_bitmap_get_nyPtr = _lookup< - ffi.NativeFunction)>>( - 'mtmd_bitmap_get_ny'); - late final _mtmd_bitmap_get_ny = _mtmd_bitmap_get_nyPtr - .asFunction)>(); + late final _mtmd_helper_video_initPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + mtmd_helper_video_init_params)>>('mtmd_helper_video_init'); + late final _mtmd_helper_video_init = _mtmd_helper_video_initPtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, mtmd_helper_video_init_params)>(); - ffi.Pointer mtmd_bitmap_get_data( - ffi.Pointer bitmap, + ffi.Pointer mtmd_helper_video_init_from_buf( + ffi.Pointer mctx, + ffi.Pointer buf, + int len, + mtmd_helper_video_init_params params, ) { - return _mtmd_bitmap_get_data( - bitmap, + return _mtmd_helper_video_init_from_buf( + mctx, + buf, + len, + params, ); } - late final _mtmd_bitmap_get_dataPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('mtmd_bitmap_get_data'); - late final _mtmd_bitmap_get_data = _mtmd_bitmap_get_dataPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + late final _mtmd_helper_video_init_from_bufPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + ffi.Size, + mtmd_helper_video_init_params)>>( + 'mtmd_helper_video_init_from_buf'); + late final _mtmd_helper_video_init_from_buf = + _mtmd_helper_video_init_from_bufPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + int, + mtmd_helper_video_init_params)>(); - int mtmd_bitmap_get_n_bytes( - ffi.Pointer bitmap, - ) { - return _mtmd_bitmap_get_n_bytes( - bitmap, - ); + mtmd_helper_video_init_params mtmd_helper_video_init_params_default() { + return _mtmd_helper_video_init_params_default(); } - late final _mtmd_bitmap_get_n_bytesPtr = - _lookup)>>( - 'mtmd_bitmap_get_n_bytes'); - late final _mtmd_bitmap_get_n_bytes = _mtmd_bitmap_get_n_bytesPtr - .asFunction)>(); + late final _mtmd_helper_video_init_params_defaultPtr = + _lookup>( + 'mtmd_helper_video_init_params_default'); + late final _mtmd_helper_video_init_params_default = + _mtmd_helper_video_init_params_defaultPtr + .asFunction(); - bool mtmd_bitmap_is_audio( - ffi.Pointer bitmap, + int mtmd_helper_video_read_next( + ffi.Pointer ctx, + ffi.Pointer> out_bitmap, + ffi.Pointer> out_text, ) { - return _mtmd_bitmap_is_audio( - bitmap, + return _mtmd_helper_video_read_next( + ctx, + out_bitmap, + out_text, ); } - late final _mtmd_bitmap_is_audioPtr = - _lookup)>>( - 'mtmd_bitmap_is_audio'); - late final _mtmd_bitmap_is_audio = _mtmd_bitmap_is_audioPtr - .asFunction)>(); + late final _mtmd_helper_video_read_nextPtr = _lookup< + ffi.NativeFunction< + ffi.Int32 Function( + ffi.Pointer, + ffi.Pointer>, + ffi.Pointer>)>>( + 'mtmd_helper_video_read_next'); + late final _mtmd_helper_video_read_next = + _mtmd_helper_video_read_nextPtr.asFunction< + int Function( + ffi.Pointer, + ffi.Pointer>, + ffi.Pointer>)>(); - void mtmd_bitmap_free( - ffi.Pointer bitmap, + mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos( + ffi.Pointer image_tokens, + int pos_0, + int i, ) { - return _mtmd_bitmap_free( - bitmap, + return _mtmd_image_tokens_get_decoder_pos( + image_tokens, + pos_0, + i, ); } - late final _mtmd_bitmap_freePtr = - _lookup)>>( - 'mtmd_bitmap_free'); - late final _mtmd_bitmap_free = _mtmd_bitmap_freePtr - .asFunction)>(); + late final _mtmd_image_tokens_get_decoder_posPtr = _lookup< + ffi.NativeFunction< + mtmd_decoder_pos Function(ffi.Pointer, llama_pos, + ffi.Size)>>('mtmd_image_tokens_get_decoder_pos'); + late final _mtmd_image_tokens_get_decoder_pos = + _mtmd_image_tokens_get_decoder_posPtr.asFunction< + mtmd_decoder_pos Function( + ffi.Pointer, int, int)>(); - ffi.Pointer mtmd_bitmap_get_id( - ffi.Pointer bitmap, + ffi.Pointer mtmd_image_tokens_get_id( + ffi.Pointer image_tokens, ) { - return _mtmd_bitmap_get_id( - bitmap, + return _mtmd_image_tokens_get_id( + image_tokens, ); } - late final _mtmd_bitmap_get_idPtr = _lookup< + late final _mtmd_image_tokens_get_idPtr = _lookup< ffi.NativeFunction< ffi.Pointer Function( - ffi.Pointer)>>('mtmd_bitmap_get_id'); - late final _mtmd_bitmap_get_id = _mtmd_bitmap_get_idPtr - .asFunction Function(ffi.Pointer)>(); + ffi.Pointer)>>('mtmd_image_tokens_get_id'); + late final _mtmd_image_tokens_get_id = + _mtmd_image_tokens_get_idPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - void mtmd_bitmap_set_id( - ffi.Pointer bitmap, - ffi.Pointer id, + int mtmd_image_tokens_get_n_pos( + ffi.Pointer image_tokens, ) { - return _mtmd_bitmap_set_id( - bitmap, - id, + return _mtmd_image_tokens_get_n_pos( + image_tokens, ); } - late final _mtmd_bitmap_set_idPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, - ffi.Pointer)>>('mtmd_bitmap_set_id'); - late final _mtmd_bitmap_set_id = _mtmd_bitmap_set_idPtr.asFunction< - void Function(ffi.Pointer, ffi.Pointer)>(); + late final _mtmd_image_tokens_get_n_posPtr = _lookup< + ffi + .NativeFunction)>>( + 'mtmd_image_tokens_get_n_pos'); + late final _mtmd_image_tokens_get_n_pos = _mtmd_image_tokens_get_n_posPtr + .asFunction)>(); - ffi.Pointer mtmd_bitmap_init_lazy( - ffi.Pointer ctx, - ffi.Pointer id, - ffi.Pointer user_data, - mtmd_bitmap_lazy_callback callback, + int mtmd_image_tokens_get_n_tokens( + ffi.Pointer image_tokens, ) { - return _mtmd_bitmap_init_lazy( - ctx, - id, - user_data, - callback, + return _mtmd_image_tokens_get_n_tokens( + image_tokens, ); } - late final _mtmd_bitmap_init_lazyPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - mtmd_bitmap_lazy_callback)>>('mtmd_bitmap_init_lazy'); - late final _mtmd_bitmap_init_lazy = _mtmd_bitmap_init_lazyPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - mtmd_bitmap_lazy_callback)>(); + late final _mtmd_image_tokens_get_n_tokensPtr = _lookup< + ffi + .NativeFunction)>>( + 'mtmd_image_tokens_get_n_tokens'); + late final _mtmd_image_tokens_get_n_tokens = + _mtmd_image_tokens_get_n_tokensPtr + .asFunction)>(); - ffi.Pointer mtmd_input_chunks_init() { - return _mtmd_input_chunks_init(); + @Deprecated('use mtmd_image_tokens_get_decoder_pos() instead') + int mtmd_image_tokens_get_nx( + ffi.Pointer image_tokens, + ) { + return _mtmd_image_tokens_get_nx( + image_tokens, + ); } - late final _mtmd_input_chunks_initPtr = - _lookup Function()>>( - 'mtmd_input_chunks_init'); - late final _mtmd_input_chunks_init = _mtmd_input_chunks_initPtr - .asFunction Function()>(); + late final _mtmd_image_tokens_get_nxPtr = _lookup< + ffi + .NativeFunction)>>( + 'mtmd_image_tokens_get_nx'); + late final _mtmd_image_tokens_get_nx = _mtmd_image_tokens_get_nxPtr + .asFunction)>(); - int mtmd_input_chunks_size( - ffi.Pointer chunks, + @Deprecated('use mtmd_image_tokens_get_decoder_pos() instead') + int mtmd_image_tokens_get_ny( + ffi.Pointer image_tokens, ) { - return _mtmd_input_chunks_size( - chunks, + return _mtmd_image_tokens_get_ny( + image_tokens, ); } - late final _mtmd_input_chunks_sizePtr = _lookup< + late final _mtmd_image_tokens_get_nyPtr = _lookup< ffi - .NativeFunction)>>( - 'mtmd_input_chunks_size'); - late final _mtmd_input_chunks_size = _mtmd_input_chunks_sizePtr - .asFunction)>(); + .NativeFunction)>>( + 'mtmd_image_tokens_get_ny'); + late final _mtmd_image_tokens_get_ny = _mtmd_image_tokens_get_nyPtr + .asFunction)>(); - ffi.Pointer mtmd_input_chunks_get( - ffi.Pointer chunks, - int idx, + ffi.Pointer mtmd_init_from_file( + ffi.Pointer mmproj_fname, + ffi.Pointer text_model, + mtmd_context_params ctx_params, ) { - return _mtmd_input_chunks_get( - chunks, - idx, + return _mtmd_init_from_file( + mmproj_fname, + text_model, + ctx_params, ); } - late final _mtmd_input_chunks_getPtr = _lookup< + late final _mtmd_init_from_filePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Size)>>('mtmd_input_chunks_get'); - late final _mtmd_input_chunks_get = _mtmd_input_chunks_getPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, int)>(); + ffi.Pointer Function( + ffi.Pointer, + ffi.Pointer, + mtmd_context_params)>>('mtmd_init_from_file'); + late final _mtmd_init_from_file = _mtmd_init_from_filePtr.asFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer, mtmd_context_params)>(); - void mtmd_input_chunks_free( - ffi.Pointer chunks, + ffi.Pointer mtmd_input_chunk_copy( + ffi.Pointer chunk, ) { - return _mtmd_input_chunks_free( - chunks, + return _mtmd_input_chunk_copy( + chunk, ); } - late final _mtmd_input_chunks_freePtr = _lookup< - ffi - .NativeFunction)>>( - 'mtmd_input_chunks_free'); - late final _mtmd_input_chunks_free = _mtmd_input_chunks_freePtr - .asFunction)>(); + late final _mtmd_input_chunk_copyPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer)>>('mtmd_input_chunk_copy'); + late final _mtmd_input_chunk_copy = _mtmd_input_chunk_copyPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - mtmd_input_chunk_type mtmd_input_chunk_get_type( + void mtmd_input_chunk_free( ffi.Pointer chunk, ) { - return mtmd_input_chunk_type.fromValue(_mtmd_input_chunk_get_type( + return _mtmd_input_chunk_free( chunk, - )); + ); } - late final _mtmd_input_chunk_get_typePtr = _lookup< - ffi.NativeFunction< - ffi.UnsignedInt Function( - ffi.Pointer)>>('mtmd_input_chunk_get_type'); - late final _mtmd_input_chunk_get_type = _mtmd_input_chunk_get_typePtr - .asFunction)>(); + late final _mtmd_input_chunk_freePtr = _lookup< + ffi.NativeFunction)>>( + 'mtmd_input_chunk_free'); + late final _mtmd_input_chunk_free = _mtmd_input_chunk_freePtr + .asFunction)>(); - ffi.Pointer mtmd_input_chunk_get_tokens_text( + ffi.Pointer mtmd_input_chunk_get_id( ffi.Pointer chunk, - ffi.Pointer n_tokens_output, ) { - return _mtmd_input_chunk_get_tokens_text( + return _mtmd_input_chunk_get_id( chunk, - n_tokens_output, ); } - late final _mtmd_input_chunk_get_tokens_textPtr = _lookup< + late final _mtmd_input_chunk_get_idPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer)>>('mtmd_input_chunk_get_tokens_text'); - late final _mtmd_input_chunk_get_tokens_text = - _mtmd_input_chunk_get_tokens_textPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function( + ffi.Pointer)>>('mtmd_input_chunk_get_id'); + late final _mtmd_input_chunk_get_id = _mtmd_input_chunk_get_idPtr.asFunction< + ffi.Pointer Function(ffi.Pointer)>(); - ffi.Pointer mtmd_input_chunk_get_tokens_image( + int mtmd_input_chunk_get_n_pos( ffi.Pointer chunk, ) { - return _mtmd_input_chunk_get_tokens_image( + return _mtmd_input_chunk_get_n_pos( chunk, ); } - late final _mtmd_input_chunk_get_tokens_imagePtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>( - 'mtmd_input_chunk_get_tokens_image'); - late final _mtmd_input_chunk_get_tokens_image = - _mtmd_input_chunk_get_tokens_imagePtr.asFunction< - ffi.Pointer Function( - ffi.Pointer)>(); + late final _mtmd_input_chunk_get_n_posPtr = _lookup< + ffi + .NativeFunction)>>( + 'mtmd_input_chunk_get_n_pos'); + late final _mtmd_input_chunk_get_n_pos = _mtmd_input_chunk_get_n_posPtr + .asFunction)>(); int mtmd_input_chunk_get_n_tokens( ffi.Pointer chunk, @@ -19266,162 +19501,171 @@ class LlamaBindings { late final _mtmd_input_chunk_get_n_tokens = _mtmd_input_chunk_get_n_tokensPtr .asFunction)>(); - ffi.Pointer mtmd_input_chunk_get_id( + ffi.Pointer mtmd_input_chunk_get_tokens_image( ffi.Pointer chunk, ) { - return _mtmd_input_chunk_get_id( + return _mtmd_input_chunk_get_tokens_image( chunk, ); } - late final _mtmd_input_chunk_get_idPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('mtmd_input_chunk_get_id'); - late final _mtmd_input_chunk_get_id = _mtmd_input_chunk_get_idPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + late final _mtmd_input_chunk_get_tokens_imagePtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer)>>( + 'mtmd_input_chunk_get_tokens_image'); + late final _mtmd_input_chunk_get_tokens_image = + _mtmd_input_chunk_get_tokens_imagePtr.asFunction< + ffi.Pointer Function( + ffi.Pointer)>(); - int mtmd_input_chunk_get_n_pos( + ffi.Pointer mtmd_input_chunk_get_tokens_text( ffi.Pointer chunk, + ffi.Pointer n_tokens_output, ) { - return _mtmd_input_chunk_get_n_pos( + return _mtmd_input_chunk_get_tokens_text( chunk, + n_tokens_output, ); } - late final _mtmd_input_chunk_get_n_posPtr = _lookup< - ffi - .NativeFunction)>>( - 'mtmd_input_chunk_get_n_pos'); - late final _mtmd_input_chunk_get_n_pos = _mtmd_input_chunk_get_n_posPtr - .asFunction)>(); + late final _mtmd_input_chunk_get_tokens_textPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer)>>('mtmd_input_chunk_get_tokens_text'); + late final _mtmd_input_chunk_get_tokens_text = + _mtmd_input_chunk_get_tokens_textPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - ffi.Pointer mtmd_input_chunk_copy( + mtmd_input_chunk_type mtmd_input_chunk_get_type( ffi.Pointer chunk, ) { - return _mtmd_input_chunk_copy( + return mtmd_input_chunk_type.fromValue(_mtmd_input_chunk_get_type( chunk, - ); + )); } - late final _mtmd_input_chunk_copyPtr = _lookup< + late final _mtmd_input_chunk_get_typePtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('mtmd_input_chunk_copy'); - late final _mtmd_input_chunk_copy = _mtmd_input_chunk_copyPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + ffi.UnsignedInt Function( + ffi.Pointer)>>('mtmd_input_chunk_get_type'); + late final _mtmd_input_chunk_get_type = _mtmd_input_chunk_get_typePtr + .asFunction)>(); - void mtmd_input_chunk_free( - ffi.Pointer chunk, + void mtmd_input_chunks_free( + ffi.Pointer chunks, ) { - return _mtmd_input_chunk_free( - chunk, + return _mtmd_input_chunks_free( + chunks, ); } - late final _mtmd_input_chunk_freePtr = _lookup< - ffi.NativeFunction)>>( - 'mtmd_input_chunk_free'); - late final _mtmd_input_chunk_free = _mtmd_input_chunk_freePtr - .asFunction)>(); + late final _mtmd_input_chunks_freePtr = _lookup< + ffi + .NativeFunction)>>( + 'mtmd_input_chunks_free'); + late final _mtmd_input_chunks_free = _mtmd_input_chunks_freePtr + .asFunction)>(); - int mtmd_image_tokens_get_n_tokens( - ffi.Pointer image_tokens, + ffi.Pointer mtmd_input_chunks_get( + ffi.Pointer chunks, + int idx, ) { - return _mtmd_image_tokens_get_n_tokens( - image_tokens, + return _mtmd_input_chunks_get( + chunks, + idx, ); } - late final _mtmd_image_tokens_get_n_tokensPtr = _lookup< - ffi - .NativeFunction)>>( - 'mtmd_image_tokens_get_n_tokens'); - late final _mtmd_image_tokens_get_n_tokens = - _mtmd_image_tokens_get_n_tokensPtr - .asFunction)>(); + late final _mtmd_input_chunks_getPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer, + ffi.Size)>>('mtmd_input_chunks_get'); + late final _mtmd_input_chunks_get = _mtmd_input_chunks_getPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, int)>(); - ffi.Pointer mtmd_image_tokens_get_id( - ffi.Pointer image_tokens, - ) { - return _mtmd_image_tokens_get_id( - image_tokens, - ); + ffi.Pointer mtmd_input_chunks_init() { + return _mtmd_input_chunks_init(); } - late final _mtmd_image_tokens_get_idPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('mtmd_image_tokens_get_id'); - late final _mtmd_image_tokens_get_id = - _mtmd_image_tokens_get_idPtr.asFunction< - ffi.Pointer Function(ffi.Pointer)>(); + late final _mtmd_input_chunks_initPtr = + _lookup Function()>>( + 'mtmd_input_chunks_init'); + late final _mtmd_input_chunks_init = _mtmd_input_chunks_initPtr + .asFunction Function()>(); - int mtmd_image_tokens_get_n_pos( - ffi.Pointer image_tokens, + int mtmd_input_chunks_size( + ffi.Pointer chunks, ) { - return _mtmd_image_tokens_get_n_pos( - image_tokens, + return _mtmd_input_chunks_size( + chunks, ); } - late final _mtmd_image_tokens_get_n_posPtr = _lookup< + late final _mtmd_input_chunks_sizePtr = _lookup< ffi - .NativeFunction)>>( - 'mtmd_image_tokens_get_n_pos'); - late final _mtmd_image_tokens_get_n_pos = _mtmd_image_tokens_get_n_posPtr - .asFunction)>(); + .NativeFunction)>>( + 'mtmd_input_chunks_size'); + late final _mtmd_input_chunks_size = _mtmd_input_chunks_sizePtr + .asFunction)>(); - int mtmd_image_tokens_get_nx( - ffi.Pointer image_tokens, + void mtmd_log_set( + ggml_log_callback log_callback, + ffi.Pointer user_data, ) { - return _mtmd_image_tokens_get_nx( - image_tokens, + return _mtmd_log_set( + log_callback, + user_data, ); } - late final _mtmd_image_tokens_get_nxPtr = _lookup< - ffi - .NativeFunction)>>( - 'mtmd_image_tokens_get_nx'); - late final _mtmd_image_tokens_get_nx = _mtmd_image_tokens_get_nxPtr - .asFunction)>(); + late final _mtmd_log_setPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ggml_log_callback, ffi.Pointer)>>('mtmd_log_set'); + late final _mtmd_log_set = _mtmd_log_setPtr + .asFunction)>(); - int mtmd_image_tokens_get_ny( - ffi.Pointer image_tokens, + bool mtmd_support_audio( + ffi.Pointer ctx, ) { - return _mtmd_image_tokens_get_ny( - image_tokens, + return _mtmd_support_audio( + ctx, ); } - late final _mtmd_image_tokens_get_nyPtr = _lookup< - ffi - .NativeFunction)>>( - 'mtmd_image_tokens_get_ny'); - late final _mtmd_image_tokens_get_ny = _mtmd_image_tokens_get_nyPtr - .asFunction)>(); + late final _mtmd_support_audioPtr = + _lookup)>>( + 'mtmd_support_audio'); + late final _mtmd_support_audio = _mtmd_support_audioPtr + .asFunction)>(); - mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos( - ffi.Pointer image_tokens, - int pos_0, - int i, + bool mtmd_support_vision( + ffi.Pointer ctx, ) { - return _mtmd_image_tokens_get_decoder_pos( - image_tokens, - pos_0, - i, + return _mtmd_support_vision( + ctx, ); } - late final _mtmd_image_tokens_get_decoder_posPtr = _lookup< - ffi.NativeFunction< - mtmd_decoder_pos Function(ffi.Pointer, llama_pos, - ffi.Size)>>('mtmd_image_tokens_get_decoder_pos'); - late final _mtmd_image_tokens_get_decoder_pos = - _mtmd_image_tokens_get_decoder_posPtr.asFunction< - mtmd_decoder_pos Function( - ffi.Pointer, int, int)>(); + late final _mtmd_support_visionPtr = + _lookup)>>( + 'mtmd_support_vision'); + late final _mtmd_support_vision = _mtmd_support_visionPtr + .asFunction)>(); + + /// ////////////////////////////////////// + ffi.Pointer mtmd_test_create_input_chunks() { + return _mtmd_test_create_input_chunks(); + } + + late final _mtmd_test_create_input_chunksPtr = + _lookup Function()>>( + 'mtmd_test_create_input_chunks'); + late final _mtmd_test_create_input_chunks = _mtmd_test_create_input_chunksPtr + .asFunction Function()>(); int mtmd_tokenize( ffi.Pointer ctx, @@ -19455,4488 +19699,5423 @@ class LlamaBindings { ffi.Pointer>, int)>(); - int mtmd_encode( - ffi.Pointer ctx, - ffi.Pointer image_tokens, + ffi.Pointer open_memstream( + ffi.Pointer> __bufp, + ffi.Pointer __sizep, ) { - return _mtmd_encode( - ctx, - image_tokens, + return _open_memstream( + __bufp, + __sizep, ); } - late final _mtmd_encodePtr = _lookup< + late final _open_memstreamPtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function(ffi.Pointer, - ffi.Pointer)>>('mtmd_encode'); - late final _mtmd_encode = _mtmd_encodePtr.asFunction< - int Function( - ffi.Pointer, ffi.Pointer)>(); + ffi.Pointer Function(ffi.Pointer>, + ffi.Pointer)>>('open_memstream'); + late final _open_memstream = _open_memstreamPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer>, ffi.Pointer)>(); - int mtmd_encode_chunk( - ffi.Pointer ctx, - ffi.Pointer chunk, + int pclose( + ffi.Pointer arg0, ) { - return _mtmd_encode_chunk( - ctx, - chunk, + return _pclose( + arg0, ); } - late final _mtmd_encode_chunkPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function(ffi.Pointer, - ffi.Pointer)>>('mtmd_encode_chunk'); - late final _mtmd_encode_chunk = _mtmd_encode_chunkPtr.asFunction< - int Function(ffi.Pointer, ffi.Pointer)>(); + late final _pclosePtr = + _lookup)>>( + 'pclose'); + late final _pclose = _pclosePtr.asFunction)>(); - ffi.Pointer mtmd_get_output_embd( - ffi.Pointer ctx, + void perror( + ffi.Pointer arg0, ) { - return _mtmd_get_output_embd( - ctx, + return _perror( + arg0, ); } - late final _mtmd_get_output_embdPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer)>>('mtmd_get_output_embd'); - late final _mtmd_get_output_embd = _mtmd_get_output_embdPtr - .asFunction Function(ffi.Pointer)>(); + late final _perrorPtr = + _lookup)>>( + 'perror'); + late final _perror = + _perrorPtr.asFunction)>(); - void mtmd_log_set( - ggml_log_callback log_callback, - ffi.Pointer user_data, + ffi.Pointer popen( + ffi.Pointer arg0, + ffi.Pointer arg1, ) { - return _mtmd_log_set( - log_callback, - user_data, + return _popen( + arg0, + arg1, ); } - late final _mtmd_log_setPtr = _lookup< + late final _popenPtr = _lookup< ffi.NativeFunction< - ffi.Void Function( - ggml_log_callback, ffi.Pointer)>>('mtmd_log_set'); - late final _mtmd_log_set = _mtmd_log_setPtr - .asFunction)>(); + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>>('popen'); + late final _popen = _popenPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - mtmd_caps mtmd_get_cap_from_file( - ffi.Pointer mmproj_fname, + int printf( + ffi.Pointer arg0, ) { - return _mtmd_get_cap_from_file( - mmproj_fname, + return _printf( + arg0, ); } - late final _mtmd_get_cap_from_filePtr = - _lookup)>>( - 'mtmd_get_cap_from_file'); - late final _mtmd_get_cap_from_file = _mtmd_get_cap_from_filePtr - .asFunction)>(); + late final _printfPtr = + _lookup)>>( + 'printf'); + late final _printf = + _printfPtr.asFunction)>(); - /// ////////////////////////////////////// - ffi.Pointer mtmd_test_create_input_chunks() { - return _mtmd_test_create_input_chunks(); + int putc( + int arg0, + ffi.Pointer arg1, + ) { + return _putc( + arg0, + arg1, + ); } - late final _mtmd_test_create_input_chunksPtr = - _lookup Function()>>( - 'mtmd_test_create_input_chunks'); - late final _mtmd_test_create_input_chunks = _mtmd_test_create_input_chunksPtr - .asFunction Function()>(); + late final _putcPtr = + _lookup)>>( + 'putc'); + late final _putc = + _putcPtr.asFunction)>(); - void mtmd_helper_log_set( - ggml_log_callback log_callback, - ffi.Pointer user_data, + int putc_unlocked( + int arg0, + ffi.Pointer arg1, ) { - return _mtmd_helper_log_set( - log_callback, - user_data, + return _putc_unlocked( + arg0, + arg1, ); } - late final _mtmd_helper_log_setPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ggml_log_callback, - ffi.Pointer)>>('mtmd_helper_log_set'); - late final _mtmd_helper_log_set = _mtmd_helper_log_setPtr - .asFunction)>(); + late final _putc_unlockedPtr = + _lookup)>>( + 'putc_unlocked'); + late final _putc_unlocked = + _putc_unlockedPtr.asFunction)>(); - bool mtmd_helper_support_video( - ffi.Pointer ctx, + int putchar( + int arg0, ) { - return _mtmd_helper_support_video( - ctx, + return _putchar( + arg0, ); } - late final _mtmd_helper_support_videoPtr = - _lookup)>>( - 'mtmd_helper_support_video'); - late final _mtmd_helper_support_video = _mtmd_helper_support_videoPtr - .asFunction)>(); + late final _putcharPtr = + _lookup>('putchar'); + late final _putchar = _putcharPtr.asFunction(); - mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file( - ffi.Pointer ctx, - ffi.Pointer fname, - bool placeholder, + int putchar_unlocked( + int arg0, ) { - return _mtmd_helper_bitmap_init_from_file( - ctx, - fname, - placeholder, + return _putchar_unlocked( + arg0, ); } - late final _mtmd_helper_bitmap_init_from_filePtr = _lookup< - ffi.NativeFunction< - mtmd_helper_bitmap_wrapper Function( - ffi.Pointer, - ffi.Pointer, - ffi.Bool)>>('mtmd_helper_bitmap_init_from_file'); - late final _mtmd_helper_bitmap_init_from_file = - _mtmd_helper_bitmap_init_from_filePtr.asFunction< - mtmd_helper_bitmap_wrapper Function( - ffi.Pointer, ffi.Pointer, bool)>(); + late final _putchar_unlockedPtr = + _lookup>( + 'putchar_unlocked'); + late final _putchar_unlocked = + _putchar_unlockedPtr.asFunction(); - mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf( - ffi.Pointer ctx, - ffi.Pointer buf, - int len, - bool placeholder, + int puts( + ffi.Pointer arg0, ) { - return _mtmd_helper_bitmap_init_from_buf( - ctx, - buf, - len, - placeholder, + return _puts( + arg0, ); } - late final _mtmd_helper_bitmap_init_from_bufPtr = _lookup< - ffi.NativeFunction< - mtmd_helper_bitmap_wrapper Function( - ffi.Pointer, - ffi.Pointer, - ffi.Size, - ffi.Bool)>>('mtmd_helper_bitmap_init_from_buf'); - late final _mtmd_helper_bitmap_init_from_buf = - _mtmd_helper_bitmap_init_from_bufPtr.asFunction< - mtmd_helper_bitmap_wrapper Function(ffi.Pointer, - ffi.Pointer, int, bool)>(); + late final _putsPtr = + _lookup)>>( + 'puts'); + late final _puts = _putsPtr.asFunction)>(); - int mtmd_helper_get_n_tokens( - ffi.Pointer chunks, + int putw( + int arg0, + ffi.Pointer arg1, ) { - return _mtmd_helper_get_n_tokens( - chunks, + return _putw( + arg0, + arg1, ); } - late final _mtmd_helper_get_n_tokensPtr = _lookup< - ffi - .NativeFunction)>>( - 'mtmd_helper_get_n_tokens'); - late final _mtmd_helper_get_n_tokens = _mtmd_helper_get_n_tokensPtr - .asFunction)>(); + late final _putwPtr = + _lookup)>>( + 'putw'); + late final _putw = + _putwPtr.asFunction)>(); - int mtmd_helper_get_n_pos( - ffi.Pointer chunks, + int remove( + ffi.Pointer arg0, ) { - return _mtmd_helper_get_n_pos( - chunks, + return _remove( + arg0, ); } - late final _mtmd_helper_get_n_posPtr = _lookup< - ffi - .NativeFunction)>>( - 'mtmd_helper_get_n_pos'); - late final _mtmd_helper_get_n_pos = _mtmd_helper_get_n_posPtr - .asFunction)>(); + late final _removePtr = + _lookup)>>( + 'remove'); + late final _remove = + _removePtr.asFunction)>(); - void mtmd_helper_image_get_decoder_pos( - ffi.Pointer image, - int pos_0, - ffi.Pointer out_pos, + int rename( + ffi.Pointer __old, + ffi.Pointer __new, ) { - return _mtmd_helper_image_get_decoder_pos( - image, - pos_0, - out_pos, + return _rename( + __old, + __new, ); } - late final _mtmd_helper_image_get_decoder_posPtr = _lookup< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer, llama_pos, - ffi.Pointer)>>( - 'mtmd_helper_image_get_decoder_pos'); - late final _mtmd_helper_image_get_decoder_pos = - _mtmd_helper_image_get_decoder_posPtr.asFunction< - void Function(ffi.Pointer, int, - ffi.Pointer)>(); - - int mtmd_helper_eval_chunks( - ffi.Pointer ctx, - ffi.Pointer lctx, - ffi.Pointer chunks, - int n_past, - int seq_id, - int n_batch, - bool logits_last, - ffi.Pointer new_n_past, - ) { - return _mtmd_helper_eval_chunks( - ctx, - lctx, - chunks, - n_past, - seq_id, - n_batch, - logits_last, - new_n_past, - ); - } - - late final _mtmd_helper_eval_chunksPtr = _lookup< + late final _renamePtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - llama_pos, - llama_seq_id, - ffi.Int32, - ffi.Bool, - ffi.Pointer)>>('mtmd_helper_eval_chunks'); - late final _mtmd_helper_eval_chunks = _mtmd_helper_eval_chunksPtr.asFunction< - int Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - bool, - ffi.Pointer)>(); + ffi.Int Function( + ffi.Pointer, ffi.Pointer)>>('rename'); + late final _rename = _renamePtr + .asFunction, ffi.Pointer)>(); - int mtmd_helper_eval_chunk_single( - ffi.Pointer ctx, - ffi.Pointer lctx, - ffi.Pointer chunk, - int n_past, - int seq_id, - int n_batch, - bool logits_last, - ffi.Pointer new_n_past, + int renameat( + int arg0, + ffi.Pointer arg1, + int arg2, + ffi.Pointer arg3, ) { - return _mtmd_helper_eval_chunk_single( - ctx, - lctx, - chunk, - n_past, - seq_id, - n_batch, - logits_last, - new_n_past, + return _renameat( + arg0, + arg1, + arg2, + arg3, ); } - late final _mtmd_helper_eval_chunk_singlePtr = _lookup< + late final _renameatPtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - llama_pos, - llama_seq_id, - ffi.Int32, - ffi.Bool, - ffi.Pointer)>>('mtmd_helper_eval_chunk_single'); - late final _mtmd_helper_eval_chunk_single = - _mtmd_helper_eval_chunk_singlePtr.asFunction< - int Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - bool, - ffi.Pointer)>(); + ffi.Int Function(ffi.Int, ffi.Pointer, ffi.Int, + ffi.Pointer)>>('renameat'); + late final _renameat = _renameatPtr.asFunction< + int Function(int, ffi.Pointer, int, ffi.Pointer)>(); - int mtmd_helper_decode_image_chunk( - ffi.Pointer ctx, - ffi.Pointer lctx, - ffi.Pointer chunk, - ffi.Pointer encoded_embd, - int n_past, - int seq_id, - int n_batch, - ffi.Pointer new_n_past, + int renameatx_np( + int arg0, + ffi.Pointer arg1, + int arg2, + ffi.Pointer arg3, + int arg4, ) { - return _mtmd_helper_decode_image_chunk( - ctx, - lctx, - chunk, - encoded_embd, - n_past, - seq_id, - n_batch, - new_n_past, + return _renameatx_np( + arg0, + arg1, + arg2, + arg3, + arg4, ); } - late final _mtmd_helper_decode_image_chunkPtr = _lookup< + late final _renameatx_npPtr = _lookup< ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - llama_pos, - llama_seq_id, - ffi.Int32, - ffi.Pointer)>>('mtmd_helper_decode_image_chunk'); - late final _mtmd_helper_decode_image_chunk = - _mtmd_helper_decode_image_chunkPtr.asFunction< - int Function( - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - ffi.Pointer, - int, - int, - int, - ffi.Pointer)>(); - - mtmd_helper_video_init_params mtmd_helper_video_init_params_default() { - return _mtmd_helper_video_init_params_default(); - } - - late final _mtmd_helper_video_init_params_defaultPtr = - _lookup>( - 'mtmd_helper_video_init_params_default'); - late final _mtmd_helper_video_init_params_default = - _mtmd_helper_video_init_params_defaultPtr - .asFunction(); + ffi.Int Function(ffi.Int, ffi.Pointer, ffi.Int, + ffi.Pointer, ffi.UnsignedInt)>>('renameatx_np'); + late final _renameatx_np = _renameatx_npPtr.asFunction< + int Function( + int, ffi.Pointer, int, ffi.Pointer, int)>(); - ffi.Pointer mtmd_helper_video_init( - ffi.Pointer mctx, - ffi.Pointer path, - mtmd_helper_video_init_params params, + int renamex_np( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, ) { - return _mtmd_helper_video_init( - mctx, - path, - params, + return _renamex_np( + arg0, + arg1, + arg2, ); } - late final _mtmd_helper_video_initPtr = _lookup< + late final _renamex_npPtr = _lookup< ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - mtmd_helper_video_init_params)>>('mtmd_helper_video_init'); - late final _mtmd_helper_video_init = _mtmd_helper_video_initPtr.asFunction< - ffi.Pointer Function(ffi.Pointer, - ffi.Pointer, mtmd_helper_video_init_params)>(); + ffi.Int Function(ffi.Pointer, ffi.Pointer, + ffi.UnsignedInt)>>('renamex_np'); + late final _renamex_np = _renamex_npPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int)>(); - ffi.Pointer mtmd_helper_video_init_from_buf( - ffi.Pointer mctx, - ffi.Pointer buf, - int len, - mtmd_helper_video_init_params params, + void rewind( + ffi.Pointer arg0, ) { - return _mtmd_helper_video_init_from_buf( - mctx, - buf, - len, - params, + return _rewind( + arg0, ); } - late final _mtmd_helper_video_init_from_bufPtr = _lookup< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - ffi.Size, - mtmd_helper_video_init_params)>>( - 'mtmd_helper_video_init_from_buf'); - late final _mtmd_helper_video_init_from_buf = - _mtmd_helper_video_init_from_bufPtr.asFunction< - ffi.Pointer Function( - ffi.Pointer, - ffi.Pointer, - int, - mtmd_helper_video_init_params)>(); + late final _rewindPtr = + _lookup)>>( + 'rewind'); + late final _rewind = + _rewindPtr.asFunction)>(); - void mtmd_helper_video_free( - ffi.Pointer ctx, + int scanf( + ffi.Pointer arg0, ) { - return _mtmd_helper_video_free( - ctx, + return _scanf( + arg0, ); } - late final _mtmd_helper_video_freePtr = _lookup< - ffi - .NativeFunction)>>( - 'mtmd_helper_video_free'); - late final _mtmd_helper_video_free = _mtmd_helper_video_freePtr - .asFunction)>(); + late final _scanfPtr = + _lookup)>>( + 'scanf'); + late final _scanf = + _scanfPtr.asFunction)>(); - mtmd_helper_video_info mtmd_helper_video_get_info( - ffi.Pointer ctx, + void setbuf( + ffi.Pointer arg0, + ffi.Pointer arg1, ) { - return _mtmd_helper_video_get_info( - ctx, + return _setbuf( + arg0, + arg1, ); } - late final _mtmd_helper_video_get_infoPtr = _lookup< + late final _setbufPtr = _lookup< ffi.NativeFunction< - mtmd_helper_video_info Function( - ffi.Pointer)>>('mtmd_helper_video_get_info'); - late final _mtmd_helper_video_get_info = - _mtmd_helper_video_get_infoPtr.asFunction< - mtmd_helper_video_info Function(ffi.Pointer)>(); + ffi.Void Function( + ffi.Pointer, ffi.Pointer)>>('setbuf'); + late final _setbuf = _setbufPtr + .asFunction, ffi.Pointer)>(); - int mtmd_helper_video_read_next( - ffi.Pointer ctx, - ffi.Pointer> out_bitmap, - ffi.Pointer> out_text, + void setbuffer( + ffi.Pointer arg0, + ffi.Pointer arg1, + int __size, ) { - return _mtmd_helper_video_read_next( - ctx, - out_bitmap, - out_text, + return _setbuffer( + arg0, + arg1, + __size, ); } - late final _mtmd_helper_video_read_nextPtr = _lookup< - ffi.NativeFunction< - ffi.Int32 Function( - ffi.Pointer, - ffi.Pointer>, - ffi.Pointer>)>>( - 'mtmd_helper_video_read_next'); - late final _mtmd_helper_video_read_next = - _mtmd_helper_video_read_nextPtr.asFunction< - int Function( - ffi.Pointer, - ffi.Pointer>, - ffi.Pointer>)>(); -} - -typedef __int8_t = ffi.SignedChar; -typedef Dart__int8_t = int; -typedef __uint8_t = ffi.UnsignedChar; -typedef Dart__uint8_t = int; -typedef __int16_t = ffi.Short; -typedef Dart__int16_t = int; -typedef __uint16_t = ffi.UnsignedShort; -typedef Dart__uint16_t = int; -typedef __int32_t = ffi.Int; -typedef Dart__int32_t = int; -typedef __uint32_t = ffi.UnsignedInt; -typedef Dart__uint32_t = int; -typedef __int64_t = ffi.LongLong; -typedef Dart__int64_t = int; -typedef __uint64_t = ffi.UnsignedLongLong; -typedef Dart__uint64_t = int; -typedef __darwin_intptr_t = ffi.Long; -typedef Dart__darwin_intptr_t = int; -typedef __darwin_natural_t = ffi.UnsignedInt; -typedef Dart__darwin_natural_t = int; -typedef __darwin_ct_rune_t = ffi.Int; -typedef Dart__darwin_ct_rune_t = int; + late final _setbufferPtr = _lookup< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer, ffi.Pointer, ffi.Int)>>('setbuffer'); + late final _setbuffer = _setbufferPtr.asFunction< + void Function(ffi.Pointer, ffi.Pointer, int)>(); -final class __mbstate_t extends ffi.Union { - @ffi.Array.multi([128]) - external ffi.Array __mbstate8; + int setlinebuf( + ffi.Pointer arg0, + ) { + return _setlinebuf( + arg0, + ); + } - @ffi.LongLong() - external int _mbstateL; -} + late final _setlinebufPtr = + _lookup)>>( + 'setlinebuf'); + late final _setlinebuf = + _setlinebufPtr.asFunction)>(); -typedef __darwin_mbstate_t = __mbstate_t; -typedef __darwin_ptrdiff_t = ffi.Long; -typedef Dart__darwin_ptrdiff_t = int; -typedef __darwin_size_t = ffi.UnsignedLong; -typedef Dart__darwin_size_t = int; -typedef __builtin_va_list = ffi.Pointer; -typedef __darwin_va_list = __builtin_va_list; -typedef __darwin_wchar_t = ffi.Int; -typedef Dart__darwin_wchar_t = int; -typedef __darwin_rune_t = __darwin_wchar_t; -typedef __darwin_wint_t = ffi.Int; -typedef Dart__darwin_wint_t = int; -typedef __darwin_clock_t = ffi.UnsignedLong; -typedef Dart__darwin_clock_t = int; -typedef __darwin_socklen_t = __uint32_t; -typedef __darwin_ssize_t = ffi.Long; -typedef Dart__darwin_ssize_t = int; -typedef __darwin_time_t = ffi.Long; -typedef Dart__darwin_time_t = int; -typedef __darwin_blkcnt_t = __int64_t; -typedef __darwin_blksize_t = __int32_t; -typedef __darwin_dev_t = __int32_t; -typedef __darwin_fsblkcnt_t = ffi.UnsignedInt; -typedef Dart__darwin_fsblkcnt_t = int; -typedef __darwin_fsfilcnt_t = ffi.UnsignedInt; -typedef Dart__darwin_fsfilcnt_t = int; -typedef __darwin_gid_t = __uint32_t; -typedef __darwin_id_t = __uint32_t; -typedef __darwin_ino64_t = __uint64_t; -typedef __darwin_ino_t = __darwin_ino64_t; -typedef __darwin_mach_port_name_t = __darwin_natural_t; -typedef __darwin_mach_port_t = __darwin_mach_port_name_t; -typedef __darwin_mode_t = __uint16_t; -typedef __darwin_off_t = __int64_t; -typedef __darwin_pid_t = __int32_t; -typedef __darwin_sigset_t = __uint32_t; -typedef __darwin_suseconds_t = __int32_t; -typedef __darwin_uid_t = __uint32_t; -typedef __darwin_useconds_t = __uint32_t; + int setvbuf( + ffi.Pointer arg0, + ffi.Pointer arg1, + int arg2, + int __size, + ) { + return _setvbuf( + arg0, + arg1, + arg2, + __size, + ); + } -final class __darwin_pthread_handler_rec extends ffi.Struct { - external ffi - .Pointer)>> - __routine; + late final _setvbufPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, ffi.Int, + ffi.Size)>>('setvbuf'); + late final _setvbuf = _setvbufPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, int, int)>(); - external ffi.Pointer __arg; + int snprintf( + ffi.Pointer __str, + int __size, + ffi.Pointer __format, + ) { + return _snprintf( + __str, + __size, + __format, + ); + } - external ffi.Pointer<__darwin_pthread_handler_rec> __next; -} + late final _snprintfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Size, + ffi.Pointer)>>('snprintf'); + late final _snprintf = _snprintfPtr.asFunction< + int Function(ffi.Pointer, int, ffi.Pointer)>(); -final class _opaque_pthread_attr_t extends ffi.Struct { - @ffi.Long() - external int __sig; + @Deprecated( + 'This function is provided for compatibility reasons only. Due to security concerns inherent in the design of sprintf(3), it is highly recommended that you use snprintf(3) instead.') + int sprintf( + ffi.Pointer arg0, + ffi.Pointer arg1, + ) { + return _sprintf( + arg0, + arg1, + ); + } - @ffi.Array.multi([56]) - external ffi.Array __opaque; -} + late final _sprintfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer)>>('sprintf'); + late final _sprintf = _sprintfPtr + .asFunction, ffi.Pointer)>(); -final class _opaque_pthread_cond_t extends ffi.Struct { - @ffi.Long() - external int __sig; + int sscanf( + ffi.Pointer arg0, + ffi.Pointer arg1, + ) { + return _sscanf( + arg0, + arg1, + ); + } - @ffi.Array.multi([40]) - external ffi.Array __opaque; -} + late final _sscanfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer)>>('sscanf'); + late final _sscanf = _sscanfPtr + .asFunction, ffi.Pointer)>(); -final class _opaque_pthread_condattr_t extends ffi.Struct { - @ffi.Long() - external int __sig; + late final ffi.Pointer>> _sys_errlist = + _lookup>>('sys_errlist'); - @ffi.Array.multi([8]) - external ffi.Array __opaque; -} + ffi.Pointer> get sys_errlist => _sys_errlist.value; -final class _opaque_pthread_mutex_t extends ffi.Struct { - @ffi.Long() - external int __sig; + set sys_errlist(ffi.Pointer> value) => + _sys_errlist.value = value; - @ffi.Array.multi([56]) - external ffi.Array __opaque; -} + late final ffi.Pointer _sys_nerr = _lookup('sys_nerr'); -final class _opaque_pthread_mutexattr_t extends ffi.Struct { - @ffi.Long() - external int __sig; + int get sys_nerr => _sys_nerr.value; - @ffi.Array.multi([8]) - external ffi.Array __opaque; -} + @Deprecated( + 'This function is provided for compatibility reasons only. Due to security concerns inherent in the design of tempnam(3), it is highly recommended that you use mkstemp(3) instead.') + ffi.Pointer tempnam( + ffi.Pointer __dir, + ffi.Pointer __prefix, + ) { + return _tempnam( + __dir, + __prefix, + ); + } -final class _opaque_pthread_once_t extends ffi.Struct { - @ffi.Long() - external int __sig; + late final _tempnamPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>>('tempnam'); + late final _tempnam = _tempnamPtr.asFunction< + ffi.Pointer Function( + ffi.Pointer, ffi.Pointer)>(); - @ffi.Array.multi([8]) - external ffi.Array __opaque; -} - -final class _opaque_pthread_rwlock_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([192]) - external ffi.Array __opaque; -} - -final class _opaque_pthread_rwlockattr_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - @ffi.Array.multi([16]) - external ffi.Array __opaque; -} - -final class _opaque_pthread_t extends ffi.Struct { - @ffi.Long() - external int __sig; - - external ffi.Pointer<__darwin_pthread_handler_rec> __cleanup_stack; - - @ffi.Array.multi([8176]) - external ffi.Array __opaque; -} - -typedef __darwin_pthread_attr_t = _opaque_pthread_attr_t; -typedef __darwin_pthread_cond_t = _opaque_pthread_cond_t; -typedef __darwin_pthread_condattr_t = _opaque_pthread_condattr_t; -typedef __darwin_pthread_key_t = ffi.UnsignedLong; -typedef Dart__darwin_pthread_key_t = int; -typedef __darwin_pthread_mutex_t = _opaque_pthread_mutex_t; -typedef __darwin_pthread_mutexattr_t = _opaque_pthread_mutexattr_t; -typedef __darwin_pthread_once_t = _opaque_pthread_once_t; -typedef __darwin_pthread_rwlock_t = _opaque_pthread_rwlock_t; -typedef __darwin_pthread_rwlockattr_t = _opaque_pthread_rwlockattr_t; -typedef __darwin_pthread_t = ffi.Pointer<_opaque_pthread_t>; -typedef __darwin_nl_item = ffi.Int; -typedef Dart__darwin_nl_item = int; -typedef __darwin_wctrans_t = ffi.Int; -typedef Dart__darwin_wctrans_t = int; -typedef __darwin_wctype_t = __uint32_t; -typedef u_int8_t = ffi.UnsignedChar; -typedef Dartu_int8_t = int; -typedef u_int16_t = ffi.UnsignedShort; -typedef Dartu_int16_t = int; -typedef u_int32_t = ffi.UnsignedInt; -typedef Dartu_int32_t = int; -typedef u_int64_t = ffi.UnsignedLongLong; -typedef Dartu_int64_t = int; -typedef register_t = ffi.Int64; -typedef Dartregister_t = int; -typedef user_addr_t = u_int64_t; -typedef user_size_t = u_int64_t; -typedef user_ssize_t = ffi.Int64; -typedef Dartuser_ssize_t = int; -typedef user_long_t = ffi.Int64; -typedef Dartuser_long_t = int; -typedef user_ulong_t = u_int64_t; -typedef user_time_t = ffi.Int64; -typedef Dartuser_time_t = int; -typedef user_off_t = ffi.Int64; -typedef Dartuser_off_t = int; -typedef syscall_arg_t = u_int64_t; -typedef ptrdiff_t = __darwin_ptrdiff_t; -typedef rsize_t = __darwin_size_t; -typedef wint_t = __darwin_wint_t; -typedef int_least8_t = ffi.Int8; -typedef Dartint_least8_t = int; -typedef int_least16_t = ffi.Int16; -typedef Dartint_least16_t = int; -typedef int_least32_t = ffi.Int32; -typedef Dartint_least32_t = int; -typedef int_least64_t = ffi.Int64; -typedef Dartint_least64_t = int; -typedef uint_least8_t = ffi.Uint8; -typedef Dartuint_least8_t = int; -typedef uint_least16_t = ffi.Uint16; -typedef Dartuint_least16_t = int; -typedef uint_least32_t = ffi.Uint32; -typedef Dartuint_least32_t = int; -typedef uint_least64_t = ffi.Uint64; -typedef Dartuint_least64_t = int; -typedef int_fast8_t = ffi.Int8; -typedef Dartint_fast8_t = int; -typedef int_fast16_t = ffi.Int16; -typedef Dartint_fast16_t = int; -typedef int_fast32_t = ffi.Int32; -typedef Dartint_fast32_t = int; -typedef int_fast64_t = ffi.Int64; -typedef Dartint_fast64_t = int; -typedef uint_fast8_t = ffi.Uint8; -typedef Dartuint_fast8_t = int; -typedef uint_fast16_t = ffi.Uint16; -typedef Dartuint_fast16_t = int; -typedef uint_fast32_t = ffi.Uint32; -typedef Dartuint_fast32_t = int; -typedef uint_fast64_t = ffi.Uint64; -typedef Dartuint_fast64_t = int; -typedef intmax_t = ffi.Long; -typedef Dartintmax_t = int; -typedef uintmax_t = ffi.UnsignedLong; -typedef Dartuintmax_t = int; -typedef va_list = __darwin_va_list; -typedef fpos_t = __darwin_off_t; - -final class __sbuf extends ffi.Struct { - external ffi.Pointer _base; - - @ffi.Int() - external int _size; -} + ffi.Pointer tmpfile() { + return _tmpfile(); + } -final class __sFILEX extends ffi.Opaque {} + late final _tmpfilePtr = + _lookup Function()>>('tmpfile'); + late final _tmpfile = _tmpfilePtr.asFunction Function()>(); -final class __sFILE extends ffi.Struct { - external ffi.Pointer _p; + @Deprecated( + 'This function is provided for compatibility reasons only. Due to security concerns inherent in the design of tmpnam(3), it is highly recommended that you use mkstemp(3) instead.') + ffi.Pointer tmpnam( + ffi.Pointer arg0, + ) { + return _tmpnam( + arg0, + ); + } - @ffi.Int() - external int _r; + late final _tmpnamPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer)>>('tmpnam'); + late final _tmpnam = _tmpnamPtr + .asFunction Function(ffi.Pointer)>(); - @ffi.Int() - external int _w; + int ungetc( + int arg0, + ffi.Pointer arg1, + ) { + return _ungetc( + arg0, + arg1, + ); + } - @ffi.Short() - external int _flags; + late final _ungetcPtr = + _lookup)>>( + 'ungetc'); + late final _ungetc = + _ungetcPtr.asFunction)>(); - @ffi.Short() - external int _file; + int vasprintf( + ffi.Pointer> arg0, + ffi.Pointer arg1, + va_list arg2, + ) { + return _vasprintf( + arg0, + arg1, + arg2, + ); + } - external __sbuf _bf; + late final _vasprintfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer>, + ffi.Pointer, va_list)>>('vasprintf'); + late final _vasprintf = _vasprintfPtr.asFunction< + int Function(ffi.Pointer>, ffi.Pointer, + va_list)>(); - @ffi.Int() - external int _lbfsize; + int vdprintf( + int arg0, + ffi.Pointer arg1, + va_list arg2, + ) { + return _vdprintf( + arg0, + arg1, + arg2, + ); + } - external ffi.Pointer _cookie; + late final _vdprintfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function( + ffi.Int, ffi.Pointer, va_list)>>('vdprintf'); + late final _vdprintf = _vdprintfPtr + .asFunction, va_list)>(); - external ffi - .Pointer)>> - _close; + int vfprintf( + ffi.Pointer arg0, + ffi.Pointer arg1, + va_list arg2, + ) { + return _vfprintf( + arg0, + arg1, + arg2, + ); + } - external ffi.Pointer< + late final _vfprintfPtr = _lookup< ffi.NativeFunction< ffi.Int Function( - ffi.Pointer, ffi.Pointer, ffi.Int)>> _read; + ffi.Pointer, ffi.Pointer, va_list)>>('vfprintf'); + late final _vfprintf = _vfprintfPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, va_list)>(); - external ffi.Pointer< - ffi.NativeFunction< - fpos_t Function(ffi.Pointer, fpos_t, ffi.Int)>> _seek; + int vfscanf( + ffi.Pointer __stream, + ffi.Pointer __format, + va_list arg2, + ) { + return _vfscanf( + __stream, + __format, + arg2, + ); + } - external ffi.Pointer< + late final _vfscanfPtr = _lookup< ffi.NativeFunction< ffi.Int Function( - ffi.Pointer, ffi.Pointer, ffi.Int)>> _write; + ffi.Pointer, ffi.Pointer, va_list)>>('vfscanf'); + late final _vfscanf = _vfscanfPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, va_list)>(); - external __sbuf _ub; + int vprintf( + ffi.Pointer arg0, + va_list arg1, + ) { + return _vprintf( + arg0, + arg1, + ); + } - external ffi.Pointer<__sFILEX> _extra; + late final _vprintfPtr = _lookup< + ffi.NativeFunction, va_list)>>( + 'vprintf'); + late final _vprintf = + _vprintfPtr.asFunction, va_list)>(); - @ffi.Int() - external int _ur; + int vscanf( + ffi.Pointer __format, + va_list arg1, + ) { + return _vscanf( + __format, + arg1, + ); + } - @ffi.Array.multi([3]) - external ffi.Array _ubuf; + late final _vscanfPtr = _lookup< + ffi.NativeFunction, va_list)>>( + 'vscanf'); + late final _vscanf = + _vscanfPtr.asFunction, va_list)>(); - @ffi.Array.multi([1]) - external ffi.Array _nbuf; + int vsnprintf( + ffi.Pointer __str, + int __size, + ffi.Pointer __format, + va_list arg3, + ) { + return _vsnprintf( + __str, + __size, + __format, + arg3, + ); + } - external __sbuf _lb; + late final _vsnprintfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Size, + ffi.Pointer, va_list)>>('vsnprintf'); + late final _vsnprintf = _vsnprintfPtr.asFunction< + int Function( + ffi.Pointer, int, ffi.Pointer, va_list)>(); - @ffi.Int() - external int _blksize; + @Deprecated( + 'This function is provided for compatibility reasons only. Due to security concerns inherent in the design of sprintf(3), it is highly recommended that you use vsnprintf(3) instead.') + int vsprintf( + ffi.Pointer arg0, + ffi.Pointer arg1, + va_list arg2, + ) { + return _vsprintf( + arg0, + arg1, + arg2, + ); + } - @fpos_t() - external int _offset; -} + late final _vsprintfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + va_list)>>('vsprintf'); + late final _vsprintf = _vsprintfPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, va_list)>(); -typedef FILE = __sFILE; -typedef off_t = __darwin_off_t; -typedef ssize_t = __darwin_ssize_t; -typedef ggml_abort_callback_tFunction = ffi.Void Function( - ffi.Pointer error_message); -typedef Dartggml_abort_callback_tFunction = void Function( - ffi.Pointer error_message); -typedef ggml_abort_callback_t - = ffi.Pointer>; + int vsscanf( + ffi.Pointer __str, + ffi.Pointer __format, + va_list arg2, + ) { + return _vsscanf( + __str, + __format, + arg2, + ); + } -enum ggml_status { - GGML_STATUS_ALLOC_FAILED(-2), - GGML_STATUS_FAILED(-1), - GGML_STATUS_SUCCESS(0), - GGML_STATUS_ABORTED(1); + late final _vsscanfPtr = _lookup< + ffi.NativeFunction< + ffi.Int Function(ffi.Pointer, ffi.Pointer, + va_list)>>('vsscanf'); + late final _vsscanf = _vsscanfPtr.asFunction< + int Function(ffi.Pointer, ffi.Pointer, va_list)>(); +} - final int value; - const ggml_status(this.value); +const int BUFSIZ = 1024; - static ggml_status fromValue(int value) => switch (value) { - -2 => GGML_STATUS_ALLOC_FAILED, - -1 => GGML_STATUS_FAILED, - 0 => GGML_STATUS_SUCCESS, - 1 => GGML_STATUS_ABORTED, - _ => throw ArgumentError('Unknown value for ggml_status: $value'), - }; -} +const int EOF = -1; -typedef ggml_fp16_t = ffi.Uint16; -typedef Dartggml_fp16_t = int; +typedef FILE = __sFILE; -final class ggml_bf16_t extends ffi.Struct { - @ffi.Uint16() - external int bits; -} +const int FILENAME_MAX = 1024; -final class ggml_object extends ffi.Opaque {} +const int FOPEN_MAX = 20; -final class ggml_context extends ffi.Opaque {} +const int GGML_BACKEND_META_MAX_DEVICES = 16; -final class ggml_cgraph extends ffi.Opaque {} +const int GGML_DEFAULT_GRAPH_SIZE = 2048; -enum ggml_type { - GGML_TYPE_F32(0), - GGML_TYPE_F16(1), - GGML_TYPE_Q4_0(2), - GGML_TYPE_Q4_1(3), - GGML_TYPE_Q5_0(6), - GGML_TYPE_Q5_1(7), - GGML_TYPE_Q8_0(8), - GGML_TYPE_Q8_1(9), - GGML_TYPE_Q2_K(10), - GGML_TYPE_Q3_K(11), - GGML_TYPE_Q4_K(12), - GGML_TYPE_Q5_K(13), - GGML_TYPE_Q6_K(14), - GGML_TYPE_Q8_K(15), - GGML_TYPE_IQ2_XXS(16), - GGML_TYPE_IQ2_XS(17), - GGML_TYPE_IQ3_XXS(18), - GGML_TYPE_IQ1_S(19), - GGML_TYPE_IQ4_NL(20), - GGML_TYPE_IQ3_S(21), - GGML_TYPE_IQ2_S(22), - GGML_TYPE_IQ4_XS(23), - GGML_TYPE_I8(24), - GGML_TYPE_I16(25), - GGML_TYPE_I32(26), - GGML_TYPE_I64(27), - GGML_TYPE_F64(28), - GGML_TYPE_IQ1_M(29), - GGML_TYPE_BF16(30), - GGML_TYPE_TQ1_0(34), - GGML_TYPE_TQ2_0(35), - GGML_TYPE_MXFP4(39), - GGML_TYPE_NVFP4(40), - GGML_TYPE_Q1_0(41), - GGML_TYPE_COUNT(42); +const int GGML_DEFAULT_N_THREADS = 4; - final int value; - const ggml_type(this.value); +const int GGML_EXIT_ABORTED = 1; - static ggml_type fromValue(int value) => switch (value) { - 0 => GGML_TYPE_F32, - 1 => GGML_TYPE_F16, - 2 => GGML_TYPE_Q4_0, - 3 => GGML_TYPE_Q4_1, - 6 => GGML_TYPE_Q5_0, - 7 => GGML_TYPE_Q5_1, - 8 => GGML_TYPE_Q8_0, - 9 => GGML_TYPE_Q8_1, - 10 => GGML_TYPE_Q2_K, - 11 => GGML_TYPE_Q3_K, - 12 => GGML_TYPE_Q4_K, - 13 => GGML_TYPE_Q5_K, - 14 => GGML_TYPE_Q6_K, - 15 => GGML_TYPE_Q8_K, - 16 => GGML_TYPE_IQ2_XXS, - 17 => GGML_TYPE_IQ2_XS, - 18 => GGML_TYPE_IQ3_XXS, - 19 => GGML_TYPE_IQ1_S, - 20 => GGML_TYPE_IQ4_NL, - 21 => GGML_TYPE_IQ3_S, - 22 => GGML_TYPE_IQ2_S, - 23 => GGML_TYPE_IQ4_XS, - 24 => GGML_TYPE_I8, - 25 => GGML_TYPE_I16, - 26 => GGML_TYPE_I32, - 27 => GGML_TYPE_I64, - 28 => GGML_TYPE_F64, - 29 => GGML_TYPE_IQ1_M, - 30 => GGML_TYPE_BF16, - 34 => GGML_TYPE_TQ1_0, - 35 => GGML_TYPE_TQ2_0, - 39 => GGML_TYPE_MXFP4, - 40 => GGML_TYPE_NVFP4, - 41 => GGML_TYPE_Q1_0, - 42 => GGML_TYPE_COUNT, - _ => throw ArgumentError('Unknown value for ggml_type: $value'), - }; -} +const int GGML_EXIT_SUCCESS = 0; -enum ggml_prec { - GGML_PREC_DEFAULT(0), - GGML_PREC_F32(10); +const int GGML_FILE_MAGIC = 1734831468; - final int value; - const ggml_prec(this.value); +const int GGML_FILE_VERSION = 2; - static ggml_prec fromValue(int value) => switch (value) { - 0 => GGML_PREC_DEFAULT, - 10 => GGML_PREC_F32, - _ => throw ArgumentError('Unknown value for ggml_prec: $value'), - }; -} +const int GGML_MAX_DIMS = 4; -enum ggml_op_hint { - GGML_HINT_NONE(0), - GGML_HINT_SRC0_IS_HADAMARD(1); +const int GGML_MAX_NAME = 64; - final int value; - const ggml_op_hint(this.value); +const int GGML_MAX_N_THREADS = 512; - static ggml_op_hint fromValue(int value) => switch (value) { - 0 => GGML_HINT_NONE, - 1 => GGML_HINT_SRC0_IS_HADAMARD, - _ => throw ArgumentError('Unknown value for ggml_op_hint: $value'), - }; -} +const int GGML_MAX_OP_PARAMS = 64; -enum ggml_ftype { - GGML_FTYPE_UNKNOWN(-1), - GGML_FTYPE_ALL_F32(0), - GGML_FTYPE_MOSTLY_F16(1), - GGML_FTYPE_MOSTLY_Q4_0(2), - GGML_FTYPE_MOSTLY_Q4_1(3), - GGML_FTYPE_MOSTLY_Q4_1_SOME_F16(4), - GGML_FTYPE_MOSTLY_Q8_0(7), - GGML_FTYPE_MOSTLY_Q5_0(8), - GGML_FTYPE_MOSTLY_Q5_1(9), - GGML_FTYPE_MOSTLY_Q2_K(10), - GGML_FTYPE_MOSTLY_Q3_K(11), - GGML_FTYPE_MOSTLY_Q4_K(12), - GGML_FTYPE_MOSTLY_Q5_K(13), - GGML_FTYPE_MOSTLY_Q6_K(14), - GGML_FTYPE_MOSTLY_IQ2_XXS(15), - GGML_FTYPE_MOSTLY_IQ2_XS(16), - GGML_FTYPE_MOSTLY_IQ3_XXS(17), - GGML_FTYPE_MOSTLY_IQ1_S(18), - GGML_FTYPE_MOSTLY_IQ4_NL(19), - GGML_FTYPE_MOSTLY_IQ3_S(20), - GGML_FTYPE_MOSTLY_IQ2_S(21), - GGML_FTYPE_MOSTLY_IQ4_XS(22), - GGML_FTYPE_MOSTLY_IQ1_M(23), - GGML_FTYPE_MOSTLY_BF16(24), - GGML_FTYPE_MOSTLY_MXFP4(25), - GGML_FTYPE_MOSTLY_NVFP4(26), - GGML_FTYPE_MOSTLY_Q1_0(27); +const int GGML_MAX_PARAMS = 2048; - final int value; - const ggml_ftype(this.value); +const int GGML_MAX_SRC = 10; - static ggml_ftype fromValue(int value) => switch (value) { - -1 => GGML_FTYPE_UNKNOWN, - 0 => GGML_FTYPE_ALL_F32, - 1 => GGML_FTYPE_MOSTLY_F16, - 2 => GGML_FTYPE_MOSTLY_Q4_0, - 3 => GGML_FTYPE_MOSTLY_Q4_1, - 4 => GGML_FTYPE_MOSTLY_Q4_1_SOME_F16, - 7 => GGML_FTYPE_MOSTLY_Q8_0, - 8 => GGML_FTYPE_MOSTLY_Q5_0, - 9 => GGML_FTYPE_MOSTLY_Q5_1, - 10 => GGML_FTYPE_MOSTLY_Q2_K, - 11 => GGML_FTYPE_MOSTLY_Q3_K, - 12 => GGML_FTYPE_MOSTLY_Q4_K, - 13 => GGML_FTYPE_MOSTLY_Q5_K, - 14 => GGML_FTYPE_MOSTLY_Q6_K, - 15 => GGML_FTYPE_MOSTLY_IQ2_XXS, - 16 => GGML_FTYPE_MOSTLY_IQ2_XS, - 17 => GGML_FTYPE_MOSTLY_IQ3_XXS, - 18 => GGML_FTYPE_MOSTLY_IQ1_S, - 19 => GGML_FTYPE_MOSTLY_IQ4_NL, - 20 => GGML_FTYPE_MOSTLY_IQ3_S, - 21 => GGML_FTYPE_MOSTLY_IQ2_S, - 22 => GGML_FTYPE_MOSTLY_IQ4_XS, - 23 => GGML_FTYPE_MOSTLY_IQ1_M, - 24 => GGML_FTYPE_MOSTLY_BF16, - 25 => GGML_FTYPE_MOSTLY_MXFP4, - 26 => GGML_FTYPE_MOSTLY_NVFP4, - 27 => GGML_FTYPE_MOSTLY_Q1_0, - _ => throw ArgumentError('Unknown value for ggml_ftype: $value'), - }; -} +const int GGML_MEM_ALIGN = 16; -enum ggml_op { - GGML_OP_NONE(0), - GGML_OP_DUP(1), - GGML_OP_ADD(2), - GGML_OP_ADD_ID(3), - GGML_OP_ADD1(4), - GGML_OP_ACC(5), - GGML_OP_SUB(6), - GGML_OP_MUL(7), - GGML_OP_DIV(8), - GGML_OP_SQR(9), - GGML_OP_SQRT(10), - GGML_OP_LOG(11), - GGML_OP_SIN(12), - GGML_OP_COS(13), - GGML_OP_SUM(14), - GGML_OP_SUM_ROWS(15), - GGML_OP_CUMSUM(16), - GGML_OP_MEAN(17), - GGML_OP_ARGMAX(18), - GGML_OP_COUNT_EQUAL(19), - GGML_OP_REPEAT(20), - GGML_OP_REPEAT_BACK(21), - GGML_OP_CONCAT(22), - GGML_OP_SILU_BACK(23), - GGML_OP_NORM(24), - GGML_OP_RMS_NORM(25), - GGML_OP_RMS_NORM_BACK(26), - GGML_OP_GROUP_NORM(27), - GGML_OP_L2_NORM(28), - GGML_OP_MUL_MAT(29), - GGML_OP_MUL_MAT_ID(30), - GGML_OP_OUT_PROD(31), - GGML_OP_SCALE(32), - GGML_OP_SET(33), - GGML_OP_CPY(34), - GGML_OP_CONT(35), - GGML_OP_RESHAPE(36), - GGML_OP_VIEW(37), - GGML_OP_PERMUTE(38), - GGML_OP_TRANSPOSE(39), - GGML_OP_GET_ROWS(40), - GGML_OP_GET_ROWS_BACK(41), - GGML_OP_SET_ROWS(42), - GGML_OP_DIAG(43), - GGML_OP_DIAG_MASK_INF(44), - GGML_OP_DIAG_MASK_ZERO(45), - GGML_OP_SOFT_MAX(46), - GGML_OP_SOFT_MAX_BACK(47), - GGML_OP_ROPE(48), - GGML_OP_ROPE_BACK(49), - GGML_OP_CLAMP(50), - GGML_OP_CONV_TRANSPOSE_1D(51), - GGML_OP_IM2COL(52), - GGML_OP_IM2COL_BACK(53), - GGML_OP_IM2COL_3D(54), - GGML_OP_COL2IM_1D(55), - GGML_OP_CONV_2D(56), - GGML_OP_CONV_3D(57), - GGML_OP_CONV_2D_DW(58), - GGML_OP_CONV_TRANSPOSE_2D(59), - GGML_OP_POOL_1D(60), - GGML_OP_POOL_2D(61), - GGML_OP_POOL_2D_BACK(62), - GGML_OP_UPSCALE(63), - GGML_OP_PAD(64), - GGML_OP_PAD_REFLECT_1D(65), - GGML_OP_ROLL(66), - GGML_OP_ARANGE(67), - GGML_OP_TIMESTEP_EMBEDDING(68), - GGML_OP_ARGSORT(69), - GGML_OP_TOP_K(70), - GGML_OP_LEAKY_RELU(71), - GGML_OP_TRI(72), - GGML_OP_FILL(73), - GGML_OP_FLASH_ATTN_EXT(74), - GGML_OP_FLASH_ATTN_BACK(75), - GGML_OP_SSM_CONV(76), - GGML_OP_SSM_SCAN(77), - GGML_OP_WIN_PART(78), - GGML_OP_WIN_UNPART(79), - GGML_OP_GET_REL_POS(80), - GGML_OP_ADD_REL_POS(81), - GGML_OP_RWKV_WKV6(82), - GGML_OP_GATED_LINEAR_ATTN(83), - GGML_OP_RWKV_WKV7(84), - GGML_OP_SOLVE_TRI(85), - GGML_OP_GATED_DELTA_NET(86), - GGML_OP_UNARY(87), - GGML_OP_MAP_CUSTOM1(88), - GGML_OP_MAP_CUSTOM2(89), - GGML_OP_MAP_CUSTOM3(90), - GGML_OP_CUSTOM(91), - GGML_OP_CROSS_ENTROPY_LOSS(92), - GGML_OP_CROSS_ENTROPY_LOSS_BACK(93), - GGML_OP_OPT_STEP_ADAMW(94), - GGML_OP_OPT_STEP_SGD(95), - GGML_OP_GLU(96), - GGML_OP_COUNT(97); +const int GGML_MROPE_SECTIONS = 4; - final int value; - const ggml_op(this.value); +const int GGML_N_TASKS_MAX = -1; - static ggml_op fromValue(int value) => switch (value) { - 0 => GGML_OP_NONE, - 1 => GGML_OP_DUP, - 2 => GGML_OP_ADD, - 3 => GGML_OP_ADD_ID, - 4 => GGML_OP_ADD1, - 5 => GGML_OP_ACC, - 6 => GGML_OP_SUB, - 7 => GGML_OP_MUL, - 8 => GGML_OP_DIV, - 9 => GGML_OP_SQR, - 10 => GGML_OP_SQRT, - 11 => GGML_OP_LOG, - 12 => GGML_OP_SIN, - 13 => GGML_OP_COS, - 14 => GGML_OP_SUM, - 15 => GGML_OP_SUM_ROWS, - 16 => GGML_OP_CUMSUM, - 17 => GGML_OP_MEAN, - 18 => GGML_OP_ARGMAX, - 19 => GGML_OP_COUNT_EQUAL, - 20 => GGML_OP_REPEAT, - 21 => GGML_OP_REPEAT_BACK, - 22 => GGML_OP_CONCAT, - 23 => GGML_OP_SILU_BACK, - 24 => GGML_OP_NORM, - 25 => GGML_OP_RMS_NORM, - 26 => GGML_OP_RMS_NORM_BACK, - 27 => GGML_OP_GROUP_NORM, - 28 => GGML_OP_L2_NORM, - 29 => GGML_OP_MUL_MAT, - 30 => GGML_OP_MUL_MAT_ID, - 31 => GGML_OP_OUT_PROD, - 32 => GGML_OP_SCALE, - 33 => GGML_OP_SET, - 34 => GGML_OP_CPY, - 35 => GGML_OP_CONT, - 36 => GGML_OP_RESHAPE, - 37 => GGML_OP_VIEW, - 38 => GGML_OP_PERMUTE, - 39 => GGML_OP_TRANSPOSE, - 40 => GGML_OP_GET_ROWS, - 41 => GGML_OP_GET_ROWS_BACK, - 42 => GGML_OP_SET_ROWS, - 43 => GGML_OP_DIAG, - 44 => GGML_OP_DIAG_MASK_INF, - 45 => GGML_OP_DIAG_MASK_ZERO, - 46 => GGML_OP_SOFT_MAX, - 47 => GGML_OP_SOFT_MAX_BACK, - 48 => GGML_OP_ROPE, - 49 => GGML_OP_ROPE_BACK, - 50 => GGML_OP_CLAMP, - 51 => GGML_OP_CONV_TRANSPOSE_1D, - 52 => GGML_OP_IM2COL, - 53 => GGML_OP_IM2COL_BACK, - 54 => GGML_OP_IM2COL_3D, - 55 => GGML_OP_COL2IM_1D, - 56 => GGML_OP_CONV_2D, - 57 => GGML_OP_CONV_3D, - 58 => GGML_OP_CONV_2D_DW, - 59 => GGML_OP_CONV_TRANSPOSE_2D, - 60 => GGML_OP_POOL_1D, - 61 => GGML_OP_POOL_2D, - 62 => GGML_OP_POOL_2D_BACK, - 63 => GGML_OP_UPSCALE, - 64 => GGML_OP_PAD, - 65 => GGML_OP_PAD_REFLECT_1D, - 66 => GGML_OP_ROLL, - 67 => GGML_OP_ARANGE, - 68 => GGML_OP_TIMESTEP_EMBEDDING, - 69 => GGML_OP_ARGSORT, - 70 => GGML_OP_TOP_K, - 71 => GGML_OP_LEAKY_RELU, - 72 => GGML_OP_TRI, - 73 => GGML_OP_FILL, - 74 => GGML_OP_FLASH_ATTN_EXT, - 75 => GGML_OP_FLASH_ATTN_BACK, - 76 => GGML_OP_SSM_CONV, - 77 => GGML_OP_SSM_SCAN, - 78 => GGML_OP_WIN_PART, - 79 => GGML_OP_WIN_UNPART, - 80 => GGML_OP_GET_REL_POS, - 81 => GGML_OP_ADD_REL_POS, - 82 => GGML_OP_RWKV_WKV6, - 83 => GGML_OP_GATED_LINEAR_ATTN, - 84 => GGML_OP_RWKV_WKV7, - 85 => GGML_OP_SOLVE_TRI, - 86 => GGML_OP_GATED_DELTA_NET, - 87 => GGML_OP_UNARY, - 88 => GGML_OP_MAP_CUSTOM1, - 89 => GGML_OP_MAP_CUSTOM2, - 90 => GGML_OP_MAP_CUSTOM3, - 91 => GGML_OP_CUSTOM, - 92 => GGML_OP_CROSS_ENTROPY_LOSS, - 93 => GGML_OP_CROSS_ENTROPY_LOSS_BACK, - 94 => GGML_OP_OPT_STEP_ADAMW, - 95 => GGML_OP_OPT_STEP_SGD, - 96 => GGML_OP_GLU, - 97 => GGML_OP_COUNT, - _ => throw ArgumentError('Unknown value for ggml_op: $value'), - }; -} +const int GGML_QNT_VERSION = 2; -enum ggml_unary_op { - GGML_UNARY_OP_ABS(0), - GGML_UNARY_OP_SGN(1), - GGML_UNARY_OP_NEG(2), - GGML_UNARY_OP_STEP(3), - GGML_UNARY_OP_TANH(4), - GGML_UNARY_OP_ELU(5), - GGML_UNARY_OP_RELU(6), - GGML_UNARY_OP_SIGMOID(7), - GGML_UNARY_OP_GELU(8), - GGML_UNARY_OP_GELU_QUICK(9), - GGML_UNARY_OP_SILU(10), - GGML_UNARY_OP_HARDSWISH(11), - GGML_UNARY_OP_HARDSIGMOID(12), - GGML_UNARY_OP_EXP(13), - GGML_UNARY_OP_EXPM1(14), - GGML_UNARY_OP_SOFTPLUS(15), - GGML_UNARY_OP_GELU_ERF(16), - GGML_UNARY_OP_XIELU(17), - GGML_UNARY_OP_FLOOR(18), - GGML_UNARY_OP_CEIL(19), - GGML_UNARY_OP_ROUND(20), - GGML_UNARY_OP_TRUNC(21), - GGML_UNARY_OP_COUNT(22); +const int GGML_QNT_VERSION_FACTOR = 1000; - final int value; - const ggml_unary_op(this.value); +const int GGML_ROPE_TYPE_IMROPE = 40; - static ggml_unary_op fromValue(int value) => switch (value) { - 0 => GGML_UNARY_OP_ABS, - 1 => GGML_UNARY_OP_SGN, - 2 => GGML_UNARY_OP_NEG, - 3 => GGML_UNARY_OP_STEP, - 4 => GGML_UNARY_OP_TANH, - 5 => GGML_UNARY_OP_ELU, - 6 => GGML_UNARY_OP_RELU, - 7 => GGML_UNARY_OP_SIGMOID, - 8 => GGML_UNARY_OP_GELU, - 9 => GGML_UNARY_OP_GELU_QUICK, - 10 => GGML_UNARY_OP_SILU, - 11 => GGML_UNARY_OP_HARDSWISH, - 12 => GGML_UNARY_OP_HARDSIGMOID, - 13 => GGML_UNARY_OP_EXP, - 14 => GGML_UNARY_OP_EXPM1, - 15 => GGML_UNARY_OP_SOFTPLUS, - 16 => GGML_UNARY_OP_GELU_ERF, - 17 => GGML_UNARY_OP_XIELU, - 18 => GGML_UNARY_OP_FLOOR, - 19 => GGML_UNARY_OP_CEIL, - 20 => GGML_UNARY_OP_ROUND, - 21 => GGML_UNARY_OP_TRUNC, - 22 => GGML_UNARY_OP_COUNT, - _ => throw ArgumentError('Unknown value for ggml_unary_op: $value'), - }; -} +const int GGML_ROPE_TYPE_MROPE = 8; -enum ggml_glu_op { - GGML_GLU_OP_REGLU(0), - GGML_GLU_OP_GEGLU(1), - GGML_GLU_OP_SWIGLU(2), - GGML_GLU_OP_SWIGLU_OAI(3), - GGML_GLU_OP_GEGLU_ERF(4), - GGML_GLU_OP_GEGLU_QUICK(5), - GGML_GLU_OP_COUNT(6); +const int GGML_ROPE_TYPE_NEOX = 2; - final int value; - const ggml_glu_op(this.value); +const int GGML_ROPE_TYPE_NORMAL = 0; - static ggml_glu_op fromValue(int value) => switch (value) { - 0 => GGML_GLU_OP_REGLU, - 1 => GGML_GLU_OP_GEGLU, - 2 => GGML_GLU_OP_SWIGLU, - 3 => GGML_GLU_OP_SWIGLU_OAI, - 4 => GGML_GLU_OP_GEGLU_ERF, - 5 => GGML_GLU_OP_GEGLU_QUICK, - 6 => GGML_GLU_OP_COUNT, - _ => throw ArgumentError('Unknown value for ggml_glu_op: $value'), - }; -} +const int GGML_ROPE_TYPE_VISION = 24; -enum ggml_object_type { - GGML_OBJECT_TYPE_TENSOR(0), - GGML_OBJECT_TYPE_GRAPH(1), - GGML_OBJECT_TYPE_WORK_BUFFER(2); +const int GGML_TENSOR_SIZE = 336; - final int value; - const ggml_object_type(this.value); +const int GGUF_DEFAULT_ALIGNMENT = 32; - static ggml_object_type fromValue(int value) => switch (value) { - 0 => GGML_OBJECT_TYPE_TENSOR, - 1 => GGML_OBJECT_TYPE_GRAPH, - 2 => GGML_OBJECT_TYPE_WORK_BUFFER, - _ => throw ArgumentError('Unknown value for ggml_object_type: $value'), - }; -} +const String GGUF_KEY_GENERAL_ALIGNMENT = 'general.alignment'; -enum ggml_log_level { - GGML_LOG_LEVEL_NONE(0), - GGML_LOG_LEVEL_DEBUG(1), - GGML_LOG_LEVEL_INFO(2), - GGML_LOG_LEVEL_WARN(3), - GGML_LOG_LEVEL_ERROR(4), - GGML_LOG_LEVEL_CONT(5); +const String GGUF_MAGIC = 'GGUF'; + +const int GGUF_VERSION = 3; + +const int INT16_MAX = 32767; + +const int INT16_MIN = -32768; + +const int INT32_MAX = 2147483647; + +const int INT32_MIN = -2147483648; + +const int INT64_MAX = 9223372036854775807; + +const int INT64_MIN = -9223372036854775808; + +const int INT8_MAX = 127; + +const int INT8_MIN = -128; + +const int INTMAX_MAX = 9223372036854775807; + +const int INTMAX_MIN = -9223372036854775808; + +const int INTPTR_MAX = 9223372036854775807; + +const int INTPTR_MIN = -9223372036854775808; + +const int INT_FAST16_MAX = 32767; + +const int INT_FAST16_MIN = -32768; + +const int INT_FAST32_MAX = 2147483647; + +const int INT_FAST32_MIN = -2147483648; + +const int INT_FAST64_MAX = 9223372036854775807; + +const int INT_FAST64_MIN = -9223372036854775808; + +const int INT_FAST8_MAX = 127; + +const int INT_FAST8_MIN = -128; + +const int INT_LEAST16_MAX = 32767; + +const int INT_LEAST16_MIN = -32768; + +const int INT_LEAST32_MAX = 2147483647; + +const int INT_LEAST32_MIN = -2147483648; + +const int INT_LEAST64_MAX = 9223372036854775807; + +const int INT_LEAST64_MIN = -9223372036854775808; + +const int INT_LEAST8_MAX = 127; + +const int INT_LEAST8_MIN = -128; + +const int LLAMA_DEFAULT_SEED = 4294967295; + +const int LLAMA_FILE_MAGIC_GGLA = 1734831201; + +const int LLAMA_FILE_MAGIC_GGSN = 1734833006; + +const int LLAMA_FILE_MAGIC_GGSQ = 1734833009; + +const int LLAMA_SESSION_MAGIC = 1734833006; + +const int LLAMA_SESSION_VERSION = 9; + +const int LLAMA_STATE_SEQ_FLAGS_NONE = 0; + +const int LLAMA_STATE_SEQ_FLAGS_ON_DEVICE = 2; + +const int LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY = 1; + +const int LLAMA_STATE_SEQ_FLAGS_SWA_ONLY = 1; + +const int LLAMA_STATE_SEQ_MAGIC = 1734833009; + +const int LLAMA_STATE_SEQ_VERSION = 2; + +const int LLAMA_TOKEN_NULL = -1; + +const int L_ctermid = 1024; + +const int L_tmpnam = 1024; + +const int MAC_OS_VERSION_11_0 = 110000; + +const int MAC_OS_VERSION_11_1 = 110100; + +const int MAC_OS_VERSION_11_3 = 110300; + +const int MAC_OS_VERSION_11_4 = 110400; + +const int MAC_OS_VERSION_11_5 = 110500; + +const int MAC_OS_VERSION_11_6 = 110600; + +const int MAC_OS_VERSION_12_0 = 120000; + +const int MAC_OS_VERSION_12_1 = 120100; + +const int MAC_OS_VERSION_12_2 = 120200; + +const int MAC_OS_VERSION_12_3 = 120300; + +const int MAC_OS_VERSION_12_4 = 120400; + +const int MAC_OS_VERSION_12_5 = 120500; + +const int MAC_OS_VERSION_12_6 = 120600; + +const int MAC_OS_VERSION_12_7 = 120700; + +const int MAC_OS_VERSION_13_0 = 130000; + +const int MAC_OS_VERSION_13_1 = 130100; + +const int MAC_OS_VERSION_13_2 = 130200; + +const int MAC_OS_VERSION_13_3 = 130300; + +const int MAC_OS_VERSION_13_4 = 130400; + +const int MAC_OS_VERSION_13_5 = 130500; + +const int MAC_OS_VERSION_13_6 = 130600; + +const int MAC_OS_VERSION_13_7 = 130700; + +const int MAC_OS_VERSION_14_0 = 140000; + +const int MAC_OS_VERSION_14_1 = 140100; + +const int MAC_OS_VERSION_14_2 = 140200; + +const int MAC_OS_VERSION_14_3 = 140300; + +const int MAC_OS_VERSION_14_4 = 140400; + +const int MAC_OS_VERSION_14_5 = 140500; + +const int MAC_OS_VERSION_14_6 = 140600; + +const int MAC_OS_VERSION_14_7 = 140700; - final int value; - const ggml_log_level(this.value); +const int MAC_OS_VERSION_15_0 = 150000; - static ggml_log_level fromValue(int value) => switch (value) { - 0 => GGML_LOG_LEVEL_NONE, - 1 => GGML_LOG_LEVEL_DEBUG, - 2 => GGML_LOG_LEVEL_INFO, - 3 => GGML_LOG_LEVEL_WARN, - 4 => GGML_LOG_LEVEL_ERROR, - 5 => GGML_LOG_LEVEL_CONT, - _ => throw ArgumentError('Unknown value for ggml_log_level: $value'), - }; -} +const int MAC_OS_VERSION_15_1 = 150100; -enum ggml_tensor_flag { - GGML_TENSOR_FLAG_INPUT(1), - GGML_TENSOR_FLAG_OUTPUT(2), - GGML_TENSOR_FLAG_PARAM(4), - GGML_TENSOR_FLAG_LOSS(8), - GGML_TENSOR_FLAG_COMPUTE(16); +const int MAC_OS_VERSION_15_2 = 150200; - final int value; - const ggml_tensor_flag(this.value); +const int MAC_OS_VERSION_15_3 = 150300; - static ggml_tensor_flag fromValue(int value) => switch (value) { - 1 => GGML_TENSOR_FLAG_INPUT, - 2 => GGML_TENSOR_FLAG_OUTPUT, - 4 => GGML_TENSOR_FLAG_PARAM, - 8 => GGML_TENSOR_FLAG_LOSS, - 16 => GGML_TENSOR_FLAG_COMPUTE, - _ => throw ArgumentError('Unknown value for ggml_tensor_flag: $value'), - }; -} +const int MAC_OS_VERSION_15_4 = 150400; -enum ggml_tri_type { - GGML_TRI_TYPE_UPPER_DIAG(0), - GGML_TRI_TYPE_UPPER(1), - GGML_TRI_TYPE_LOWER_DIAG(2), - GGML_TRI_TYPE_LOWER(3); +const int MAC_OS_VERSION_15_5 = 150500; - final int value; - const ggml_tri_type(this.value); +const int MAC_OS_VERSION_15_6 = 150600; - static ggml_tri_type fromValue(int value) => switch (value) { - 0 => GGML_TRI_TYPE_UPPER_DIAG, - 1 => GGML_TRI_TYPE_UPPER, - 2 => GGML_TRI_TYPE_LOWER_DIAG, - 3 => GGML_TRI_TYPE_LOWER, - _ => throw ArgumentError('Unknown value for ggml_tri_type: $value'), - }; -} +const int MAC_OS_VERSION_16_0 = 160000; -final class ggml_init_params extends ffi.Struct { - @ffi.Size() - external int mem_size; +const int MAC_OS_VERSION_26_0 = 260000; - external ffi.Pointer mem_buffer; +const int MAC_OS_VERSION_26_1 = 260100; - @ffi.Bool() - external bool no_alloc; -} +const int MAC_OS_VERSION_26_2 = 260200; -final class ggml_backend_buffer extends ffi.Opaque {} +const int MAC_OS_VERSION_26_3 = 260300; -final class ggml_tensor extends ffi.Struct { - @ffi.UnsignedInt() - external int typeAsInt; +const int MAC_OS_VERSION_26_4 = 260400; - ggml_type get type => ggml_type.fromValue(typeAsInt); +const int MAC_OS_VERSION_26_5 = 260500; - external ffi.Pointer buffer; +const int MAC_OS_X_VERSION_10_0 = 1000; - @ffi.Array.multi([4]) - external ffi.Array ne; +const int MAC_OS_X_VERSION_10_1 = 1010; - @ffi.Array.multi([4]) - external ffi.Array nb; +const int MAC_OS_X_VERSION_10_10 = 101000; - @ffi.UnsignedInt() - external int opAsInt; +const int MAC_OS_X_VERSION_10_10_2 = 101002; - ggml_op get op => ggml_op.fromValue(opAsInt); +const int MAC_OS_X_VERSION_10_10_3 = 101003; - @ffi.Array.multi([16]) - external ffi.Array op_params; +const int MAC_OS_X_VERSION_10_11 = 101100; - @ffi.Int32() - external int flags; +const int MAC_OS_X_VERSION_10_11_2 = 101102; - @ffi.Array.multi([10]) - external ffi.Array> src; +const int MAC_OS_X_VERSION_10_11_3 = 101103; - external ffi.Pointer view_src; +const int MAC_OS_X_VERSION_10_11_4 = 101104; - @ffi.Size() - external int view_offs; +const int MAC_OS_X_VERSION_10_12 = 101200; - external ffi.Pointer data; +const int MAC_OS_X_VERSION_10_12_1 = 101201; - @ffi.Array.multi([64]) - external ffi.Array name; +const int MAC_OS_X_VERSION_10_12_2 = 101202; - external ffi.Pointer extra; +const int MAC_OS_X_VERSION_10_12_4 = 101204; - @ffi.Array.multi([8]) - external ffi.Array padding; -} +const int MAC_OS_X_VERSION_10_13 = 101300; -typedef ggml_abort_callbackFunction = ffi.Bool Function( - ffi.Pointer data); -typedef Dartggml_abort_callbackFunction = bool Function( - ffi.Pointer data); -typedef ggml_abort_callback - = ffi.Pointer>; -typedef ggml_guid_t = ffi.Pointer>; +const int MAC_OS_X_VERSION_10_13_1 = 101301; -enum ggml_op_pool { - GGML_OP_POOL_MAX(0), - GGML_OP_POOL_AVG(1), - GGML_OP_POOL_COUNT(2); +const int MAC_OS_X_VERSION_10_13_2 = 101302; - final int value; - const ggml_op_pool(this.value); +const int MAC_OS_X_VERSION_10_13_4 = 101304; - static ggml_op_pool fromValue(int value) => switch (value) { - 0 => GGML_OP_POOL_MAX, - 1 => GGML_OP_POOL_AVG, - 2 => GGML_OP_POOL_COUNT, - _ => throw ArgumentError('Unknown value for ggml_op_pool: $value'), - }; -} +const int MAC_OS_X_VERSION_10_14 = 101400; -enum ggml_scale_mode { - GGML_SCALE_MODE_NEAREST(0), - GGML_SCALE_MODE_BILINEAR(1), - GGML_SCALE_MODE_BICUBIC(2), - GGML_SCALE_MODE_COUNT(3); +const int MAC_OS_X_VERSION_10_14_1 = 101401; - final int value; - const ggml_scale_mode(this.value); +const int MAC_OS_X_VERSION_10_14_4 = 101404; - static ggml_scale_mode fromValue(int value) => switch (value) { - 0 => GGML_SCALE_MODE_NEAREST, - 1 => GGML_SCALE_MODE_BILINEAR, - 2 => GGML_SCALE_MODE_BICUBIC, - 3 => GGML_SCALE_MODE_COUNT, - _ => throw ArgumentError('Unknown value for ggml_scale_mode: $value'), - }; -} +const int MAC_OS_X_VERSION_10_14_5 = 101405; -enum ggml_scale_flag { - GGML_SCALE_FLAG_ALIGN_CORNERS(256), - GGML_SCALE_FLAG_ANTIALIAS(512); +const int MAC_OS_X_VERSION_10_14_6 = 101406; - final int value; - const ggml_scale_flag(this.value); +const int MAC_OS_X_VERSION_10_15 = 101500; - static ggml_scale_flag fromValue(int value) => switch (value) { - 256 => GGML_SCALE_FLAG_ALIGN_CORNERS, - 512 => GGML_SCALE_FLAG_ANTIALIAS, - _ => throw ArgumentError('Unknown value for ggml_scale_flag: $value'), - }; -} +const int MAC_OS_X_VERSION_10_15_1 = 101501; -enum ggml_sort_order { - GGML_SORT_ORDER_ASC(0), - GGML_SORT_ORDER_DESC(1); +const int MAC_OS_X_VERSION_10_15_4 = 101504; - final int value; - const ggml_sort_order(this.value); +const int MAC_OS_X_VERSION_10_16 = 101600; - static ggml_sort_order fromValue(int value) => switch (value) { - 0 => GGML_SORT_ORDER_ASC, - 1 => GGML_SORT_ORDER_DESC, - _ => throw ArgumentError('Unknown value for ggml_sort_order: $value'), - }; -} +const int MAC_OS_X_VERSION_10_2 = 1020; -typedef ggml_custom1_op_tFunction = ffi.Void Function( - ffi.Pointer dst, - ffi.Pointer a, - ffi.Int ith, - ffi.Int nth, - ffi.Pointer userdata); -typedef Dartggml_custom1_op_tFunction = void Function( - ffi.Pointer dst, - ffi.Pointer a, - int ith, - int nth, - ffi.Pointer userdata); -typedef ggml_custom1_op_t - = ffi.Pointer>; -typedef ggml_custom2_op_tFunction = ffi.Void Function( - ffi.Pointer dst, - ffi.Pointer a, - ffi.Pointer b, - ffi.Int ith, - ffi.Int nth, - ffi.Pointer userdata); -typedef Dartggml_custom2_op_tFunction = void Function( - ffi.Pointer dst, - ffi.Pointer a, - ffi.Pointer b, - int ith, - int nth, - ffi.Pointer userdata); -typedef ggml_custom2_op_t - = ffi.Pointer>; -typedef ggml_custom3_op_tFunction = ffi.Void Function( - ffi.Pointer dst, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, - ffi.Int ith, - ffi.Int nth, - ffi.Pointer userdata); -typedef Dartggml_custom3_op_tFunction = void Function( - ffi.Pointer dst, - ffi.Pointer a, - ffi.Pointer b, - ffi.Pointer c, - int ith, - int nth, - ffi.Pointer userdata); -typedef ggml_custom3_op_t - = ffi.Pointer>; -typedef ggml_custom_op_tFunction = ffi.Void Function( - ffi.Pointer dst, - ffi.Int ith, - ffi.Int nth, - ffi.Pointer userdata); -typedef Dartggml_custom_op_tFunction = void Function( - ffi.Pointer dst, - int ith, - int nth, - ffi.Pointer userdata); -typedef ggml_custom_op_t - = ffi.Pointer>; -typedef ggml_log_callbackFunction = ffi.Void Function(ffi.UnsignedInt level, - ffi.Pointer text, ffi.Pointer user_data); -typedef Dartggml_log_callbackFunction = void Function(ggml_log_level level, - ffi.Pointer text, ffi.Pointer user_data); -typedef ggml_log_callback - = ffi.Pointer>; -typedef ggml_to_float_tFunction = ffi.Void Function( - ffi.Pointer x, ffi.Pointer y, ffi.Int64 k); -typedef Dartggml_to_float_tFunction = void Function( - ffi.Pointer x, ffi.Pointer y, int k); -typedef ggml_to_float_t - = ffi.Pointer>; -typedef ggml_from_float_tFunction = ffi.Void Function( - ffi.Pointer x, ffi.Pointer y, ffi.Int64 k); -typedef Dartggml_from_float_tFunction = void Function( - ffi.Pointer x, ffi.Pointer y, int k); -typedef ggml_from_float_t - = ffi.Pointer>; +const int MAC_OS_X_VERSION_10_3 = 1030; -final class ggml_type_traits extends ffi.Struct { - external ffi.Pointer type_name; +const int MAC_OS_X_VERSION_10_4 = 1040; - @ffi.Int64() - external int blck_size; +const int MAC_OS_X_VERSION_10_5 = 1050; - @ffi.Int64() - external int blck_size_interleave; +const int MAC_OS_X_VERSION_10_6 = 1060; - @ffi.Size() - external int type_size; +const int MAC_OS_X_VERSION_10_7 = 1070; - @ffi.Bool() - external bool is_quantized; +const int MAC_OS_X_VERSION_10_8 = 1080; - external ggml_to_float_t to_float; +const int MAC_OS_X_VERSION_10_9 = 1090; - external ggml_from_float_t from_float_ref; -} +const int NULL = 0; -enum ggml_sched_priority { - GGML_SCHED_PRIO_LOW(-1), - GGML_SCHED_PRIO_NORMAL(0), - GGML_SCHED_PRIO_MEDIUM(1), - GGML_SCHED_PRIO_HIGH(2), - GGML_SCHED_PRIO_REALTIME(3); +const int PTRDIFF_MAX = 9223372036854775807; - final int value; - const ggml_sched_priority(this.value); +const int PTRDIFF_MIN = -9223372036854775808; - static ggml_sched_priority fromValue(int value) => switch (value) { - -1 => GGML_SCHED_PRIO_LOW, - 0 => GGML_SCHED_PRIO_NORMAL, - 1 => GGML_SCHED_PRIO_MEDIUM, - 2 => GGML_SCHED_PRIO_HIGH, - 3 => GGML_SCHED_PRIO_REALTIME, - _ => - throw ArgumentError('Unknown value for ggml_sched_priority: $value'), - }; -} +const String P_tmpdir = '/var/tmp/'; -final class ggml_threadpool_params extends ffi.Struct { - @ffi.Array.multi([512]) - external ffi.Array cpumask; +const int RENAME_EXCL = 4; - @ffi.Int() - external int n_threads; +const int RENAME_NOFOLLOW_ANY = 16; - @ffi.Int() - external int prioAsInt; +const int RENAME_RESERVED1 = 8; - ggml_sched_priority get prio => ggml_sched_priority.fromValue(prioAsInt); +const int RENAME_RESOLVE_BENEATH = 32; - @ffi.Uint32() - external int poll; +const int RENAME_SECLUDE = 1; - @ffi.Bool() - external bool strict_cpu; +const int RENAME_SWAP = 2; - @ffi.Bool() - external bool paused; -} +const int RSIZE_MAX = 9223372036854775807; -final class ggml_threadpool extends ffi.Opaque {} +const int SEEK_CUR = 1; -typedef ggml_threadpool_t = ffi.Pointer; +const int SEEK_DATA = 4; -final class ggml_backend_buffer_type extends ffi.Opaque {} +const int SEEK_END = 2; -typedef ggml_backend_buffer_type_t = ffi.Pointer; -typedef ggml_backend_buffer_t = ffi.Pointer; +const int SEEK_HOLE = 3; -final class ggml_backend extends ffi.Opaque {} +const int SEEK_SET = 0; -typedef ggml_backend_t = ffi.Pointer; +const int SIG_ATOMIC_MAX = 2147483647; -final class ggml_tallocr extends ffi.Struct { - external ggml_backend_buffer_t buffer; +const int SIG_ATOMIC_MIN = -2147483648; - external ffi.Pointer base; +const int SIZE_MAX = -1; - @ffi.Size() - external int alignment; +const int TMP_MAX = 308915776; - @ffi.Size() - external int offset; -} +const int UINT16_MAX = 65535; -final class ggml_gallocr extends ffi.Opaque {} +const int UINT32_MAX = 4294967295; -typedef ggml_gallocr_t = ffi.Pointer; +const int UINT64_MAX = -1; -final class ggml_backend_event extends ffi.Opaque {} +const int UINT8_MAX = 255; -typedef ggml_backend_event_t = ffi.Pointer; -typedef ggml_backend_graph_plan_t = ffi.Pointer; +const int UINTMAX_MAX = -1; -final class ggml_backend_reg extends ffi.Opaque {} +const int UINTPTR_MAX = -1; -typedef ggml_backend_reg_t = ffi.Pointer; +const int UINT_FAST16_MAX = 65535; -final class ggml_backend_device extends ffi.Opaque {} +const int UINT_FAST32_MAX = 4294967295; -typedef ggml_backend_dev_t = ffi.Pointer; +const int UINT_FAST64_MAX = -1; -enum ggml_backend_buffer_usage { - GGML_BACKEND_BUFFER_USAGE_ANY(0), - GGML_BACKEND_BUFFER_USAGE_WEIGHTS(1), - GGML_BACKEND_BUFFER_USAGE_COMPUTE(2); +const int UINT_FAST8_MAX = 255; - final int value; - const ggml_backend_buffer_usage(this.value); +const int UINT_LEAST16_MAX = 65535; - static ggml_backend_buffer_usage fromValue(int value) => switch (value) { - 0 => GGML_BACKEND_BUFFER_USAGE_ANY, - 1 => GGML_BACKEND_BUFFER_USAGE_WEIGHTS, - 2 => GGML_BACKEND_BUFFER_USAGE_COMPUTE, - _ => throw ArgumentError( - 'Unknown value for ggml_backend_buffer_usage: $value'), - }; +const int UINT_LEAST32_MAX = 4294967295; + +const int UINT_LEAST64_MAX = -1; + +const int UINT_LEAST8_MAX = 255; + +const int USER_ADDR_NULL = 0; + +final class UnnamedStruct extends ffi.Struct { + @ffi.Float() + external double alpha; + + @ffi.Float() + external double beta1; + + @ffi.Float() + external double beta2; + + @ffi.Float() + external double eps; + + @ffi.Float() + external double wd; + + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required double alpha, + required double beta1, + required double beta2, + required double eps, + required double wd, + }) => + $allocator() + ..ref.alpha = alpha + ..ref.beta1 = beta1 + ..ref.beta2 = beta2 + ..ref.eps = eps + ..ref.wd = wd; } -enum ggml_backend_dev_type { - GGML_BACKEND_DEVICE_TYPE_CPU(0), - GGML_BACKEND_DEVICE_TYPE_GPU(1), - GGML_BACKEND_DEVICE_TYPE_IGPU(2), - GGML_BACKEND_DEVICE_TYPE_ACCEL(3), - GGML_BACKEND_DEVICE_TYPE_META(4); +final class UnnamedStruct$1 extends ffi.Struct { + @ffi.Float() + external double alpha; - final int value; - const ggml_backend_dev_type(this.value); + @ffi.Float() + external double wd; - static ggml_backend_dev_type fromValue(int value) => switch (value) { - 0 => GGML_BACKEND_DEVICE_TYPE_CPU, - 1 => GGML_BACKEND_DEVICE_TYPE_GPU, - 2 => GGML_BACKEND_DEVICE_TYPE_IGPU, - 3 => GGML_BACKEND_DEVICE_TYPE_ACCEL, - 4 => GGML_BACKEND_DEVICE_TYPE_META, - _ => throw ArgumentError( - 'Unknown value for ggml_backend_dev_type: $value'), - }; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required double alpha, + required double wd, + }) => + $allocator() + ..ref.alpha = alpha + ..ref.wd = wd; } -final class ggml_backend_dev_caps extends ffi.Struct { - @ffi.Bool() - external bool async; +final class UnnamedUnion extends ffi.Union { + @ffi.Int64() + external int val_i64; - @ffi.Bool() - external bool host_buffer; + @ffi.Double() + external double val_f64; @ffi.Bool() - external bool buffer_from_host_ptr; + external bool val_bool; - @ffi.Bool() - external bool events; + @ffi.Array.multi([128]) + external ffi.Array val_str; } -final class ggml_backend_dev_props extends ffi.Struct { - external ffi.Pointer name; +const int WCHAR_MAX = 2147483647; - external ffi.Pointer description; +const int WCHAR_MIN = -2147483648; - @ffi.Size() - external int memory_free; +const int WINT_MAX = 2147483647; - @ffi.Size() - external int memory_total; +const int WINT_MIN = -2147483648; - @ffi.UnsignedInt() - external int typeAsInt; +const int _DARWIN_FEATURE_64_BIT_INODE = 1; - ggml_backend_dev_type get type => ggml_backend_dev_type.fromValue(typeAsInt); +const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; - external ffi.Pointer device_id; +const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; - external ggml_backend_dev_caps caps; -} +const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; -typedef ggml_backend_comm_init_tFunction = ffi.Pointer Function( - ffi.Pointer backends, ffi.Size n_backends); -typedef Dartggml_backend_comm_init_tFunction = ffi.Pointer Function( - ffi.Pointer backends, int n_backends); -typedef ggml_backend_comm_init_t - = ffi.Pointer>; -typedef ggml_backend_comm_free_tFunction = ffi.Void Function( - ffi.Pointer comm_ctx); -typedef Dartggml_backend_comm_free_tFunction = void Function( - ffi.Pointer comm_ctx); -typedef ggml_backend_comm_free_t - = ffi.Pointer>; -typedef ggml_backend_comm_allreduce_tensor_tFunction = ffi.Bool Function( - ffi.Pointer comm_ctx, - ffi.Pointer> tensors); -typedef Dartggml_backend_comm_allreduce_tensor_tFunction = bool Function( - ffi.Pointer comm_ctx, - ffi.Pointer> tensors); -typedef ggml_backend_comm_allreduce_tensor_t = ffi - .Pointer>; -typedef ggml_backend_split_buffer_type_tFunction = ggml_backend_buffer_type_t - Function(ffi.Int main_device, ffi.Pointer tensor_split); -typedef Dartggml_backend_split_buffer_type_tFunction - = ggml_backend_buffer_type_t Function( - int main_device, ffi.Pointer tensor_split); -typedef ggml_backend_split_buffer_type_t - = ffi.Pointer>; -typedef ggml_backend_set_n_threads_tFunction = ffi.Void Function( - ggml_backend_t backend, ffi.Int n_threads); -typedef Dartggml_backend_set_n_threads_tFunction = void Function( - ggml_backend_t backend, int n_threads); -typedef ggml_backend_set_n_threads_t - = ffi.Pointer>; -typedef ggml_backend_dev_get_extra_bufts_tFunction - = ffi.Pointer Function( - ggml_backend_dev_t device); -typedef ggml_backend_dev_get_extra_bufts_t = ffi - .Pointer>; -typedef ggml_backend_set_abort_callback_tFunction = ffi.Void Function( - ggml_backend_t backend, - ggml_abort_callback abort_callback, - ffi.Pointer abort_callback_data); -typedef Dartggml_backend_set_abort_callback_tFunction = void Function( - ggml_backend_t backend, - ggml_abort_callback abort_callback, - ffi.Pointer abort_callback_data); -typedef ggml_backend_set_abort_callback_t = ffi - .Pointer>; +const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; + +const int _FORTIFY_SOURCE = 2; + +const int _IOFBF = 0; -final class ggml_backend_feature extends ffi.Struct { - external ffi.Pointer name; +const int _IOLBF = 1; - external ffi.Pointer value; -} +const int _IONBF = 2; -typedef ggml_backend_get_features_tFunction = ffi.Pointer - Function(ggml_backend_reg_t reg); -typedef ggml_backend_get_features_t - = ffi.Pointer>; +const int __API_TO_BE_DEPRECATED = 100000; -final class ggml_backend_sched extends ffi.Opaque {} +const int __API_TO_BE_DEPRECATED_DRIVERKIT = 100000; -typedef ggml_backend_sched_t = ffi.Pointer; -typedef ggml_backend_sched_eval_callbackFunction = ffi.Bool Function( - ffi.Pointer t, ffi.Bool ask, ffi.Pointer user_data); -typedef Dartggml_backend_sched_eval_callbackFunction = bool Function( - ffi.Pointer t, bool ask, ffi.Pointer user_data); -typedef ggml_backend_sched_eval_callback - = ffi.Pointer>; +const int __API_TO_BE_DEPRECATED_IOS = 100000; -enum ggml_backend_meta_split_axis { - GGML_BACKEND_SPLIT_AXIS_0(0), - GGML_BACKEND_SPLIT_AXIS_1(1), - GGML_BACKEND_SPLIT_AXIS_2(2), - GGML_BACKEND_SPLIT_AXIS_3(3), - GGML_BACKEND_SPLIT_AXIS_MIRRORED(10), - GGML_BACKEND_SPLIT_AXIS_PARTIAL(11), - GGML_BACKEND_SPLIT_AXIS_NONE(98), - GGML_BACKEND_SPLIT_AXIS_UNKNOWN(99); +const int __API_TO_BE_DEPRECATED_IOSAPPLICATIONEXTENSION = 100000; - final int value; - const ggml_backend_meta_split_axis(this.value); +const int __API_TO_BE_DEPRECATED_KERNELKIT = 100000; - static ggml_backend_meta_split_axis fromValue(int value) => switch (value) { - 0 => GGML_BACKEND_SPLIT_AXIS_0, - 1 => GGML_BACKEND_SPLIT_AXIS_1, - 2 => GGML_BACKEND_SPLIT_AXIS_2, - 3 => GGML_BACKEND_SPLIT_AXIS_3, - 10 => GGML_BACKEND_SPLIT_AXIS_MIRRORED, - 11 => GGML_BACKEND_SPLIT_AXIS_PARTIAL, - 98 => GGML_BACKEND_SPLIT_AXIS_NONE, - 99 => GGML_BACKEND_SPLIT_AXIS_UNKNOWN, - _ => throw ArgumentError( - 'Unknown value for ggml_backend_meta_split_axis: $value'), - }; -} +const int __API_TO_BE_DEPRECATED_MACCATALYST = 100000; -final class ggml_backend_meta_split_state extends ffi.Struct { - @ffi.UnsignedInt() - external int axisAsInt; +const int __API_TO_BE_DEPRECATED_MACCATALYSTAPPLICATIONEXTENSION = 100000; - ggml_backend_meta_split_axis get axis => - ggml_backend_meta_split_axis.fromValue(axisAsInt); +const int __API_TO_BE_DEPRECATED_MACOS = 100000; - @ffi.Array.multi([256]) - external ffi.Array ne; +const int __API_TO_BE_DEPRECATED_MACOSAPPLICATIONEXTENSION = 100000; - @ffi.Array.multi([16]) - external ffi.Array nr; +const int __API_TO_BE_DEPRECATED_TVOS = 100000; - @ffi.Uint32() - external int n_segments; -} +const int __API_TO_BE_DEPRECATED_TVOSAPPLICATIONEXTENSION = 100000; -typedef ggml_backend_meta_get_split_state_tFunction - = ggml_backend_meta_split_state Function( - ffi.Pointer tensor, ffi.Pointer userdata); -typedef ggml_backend_meta_get_split_state_t = ffi - .Pointer>; +const int __API_TO_BE_DEPRECATED_VISIONOS = 100000; -final class ggml_backend_graph_copy$1 extends ffi.Struct { - external ggml_backend_buffer_t buffer; +const int __API_TO_BE_DEPRECATED_VISIONOSAPPLICATIONEXTENSION = 100000; - external ffi.Pointer ctx_allocated; +const int __API_TO_BE_DEPRECATED_WATCHOS = 100000; - external ffi.Pointer ctx_unallocated; +const int __API_TO_BE_DEPRECATED_WATCHOSAPPLICATIONEXTENSION = 100000; - external ffi.Pointer graph; -} +const String __AVAILABILITY_FILE = 'AvailabilityVersions.h'; -typedef ggml_backend_eval_callbackFunction = ffi.Bool Function( - ffi.Int node_index, - ffi.Pointer t1, - ffi.Pointer t2, - ffi.Pointer user_data); -typedef Dartggml_backend_eval_callbackFunction = bool Function( - int node_index, - ffi.Pointer t1, - ffi.Pointer t2, - ffi.Pointer user_data); -typedef ggml_backend_eval_callback - = ffi.Pointer>; +const int __AVAILABILITY_VERSIONS_VERSION_HASH = 93585900; -final class ggml_cplan extends ffi.Struct { - @ffi.Size() - external int work_size; +const String __AVAILABILITY_VERSIONS_VERSION_STRING = 'Local'; - external ffi.Pointer work_data; +const int __BRIDGEOS_10_0 = 100000; - @ffi.Int() - external int n_threads; +const int __BRIDGEOS_10_1 = 100100; - external ffi.Pointer threadpool; +const int __BRIDGEOS_10_2 = 100200; - external ggml_abort_callback abort_callback; +const int __BRIDGEOS_10_3 = 100300; - external ffi.Pointer abort_callback_data; +const int __BRIDGEOS_10_4 = 100400; - @ffi.Bool() - external bool use_ref; -} +const int __BRIDGEOS_26_5 = 260500; -enum ggml_numa_strategy { - GGML_NUMA_STRATEGY_DISABLED(0), - GGML_NUMA_STRATEGY_DISTRIBUTE(1), - GGML_NUMA_STRATEGY_ISOLATE(2), - GGML_NUMA_STRATEGY_NUMACTL(3), - GGML_NUMA_STRATEGY_MIRROR(4), - GGML_NUMA_STRATEGY_COUNT(5); +const int __BRIDGEOS_2_0 = 20000; - final int value; - const ggml_numa_strategy(this.value); +const int __BRIDGEOS_3_0 = 30000; - static ggml_numa_strategy fromValue(int value) => switch (value) { - 0 => GGML_NUMA_STRATEGY_DISABLED, - 1 => GGML_NUMA_STRATEGY_DISTRIBUTE, - 2 => GGML_NUMA_STRATEGY_ISOLATE, - 3 => GGML_NUMA_STRATEGY_NUMACTL, - 4 => GGML_NUMA_STRATEGY_MIRROR, - 5 => GGML_NUMA_STRATEGY_COUNT, - _ => - throw ArgumentError('Unknown value for ggml_numa_strategy: $value'), - }; -} +const int __BRIDGEOS_3_1 = 30100; -typedef ggml_vec_dot_tFunction = ffi.Void Function( - ffi.Int n, - ffi.Pointer s, - ffi.Size bs, - ffi.Pointer x, - ffi.Size bx, - ffi.Pointer y, - ffi.Size by, - ffi.Int nrc); -typedef Dartggml_vec_dot_tFunction = void Function( - int n, - ffi.Pointer s, - int bs, - ffi.Pointer x, - int bx, - ffi.Pointer y, - int by, - int nrc); -typedef ggml_vec_dot_t - = ffi.Pointer>; +const int __BRIDGEOS_3_4 = 30400; -final class ggml_type_traits_cpu extends ffi.Struct { - external ggml_from_float_t from_float; +const int __BRIDGEOS_4_0 = 40000; - external ggml_vec_dot_t vec_dot; +const int __BRIDGEOS_4_1 = 40100; - @ffi.UnsignedInt() - external int vec_dot_typeAsInt; +const int __BRIDGEOS_5_0 = 50000; - ggml_type get vec_dot_type => ggml_type.fromValue(vec_dot_typeAsInt); +const int __BRIDGEOS_5_1 = 50100; - @ffi.Int64() - external int nrows; -} +const int __BRIDGEOS_5_3 = 50300; -final class ggml_opt_dataset extends ffi.Opaque {} +const int __BRIDGEOS_6_0 = 60000; -final class ggml_opt_context extends ffi.Opaque {} +const int __BRIDGEOS_6_2 = 60200; -final class ggml_opt_result extends ffi.Opaque {} +const int __BRIDGEOS_6_4 = 60400; -typedef ggml_opt_dataset_t = ffi.Pointer; -typedef ggml_opt_context_t = ffi.Pointer; -typedef ggml_opt_result_t = ffi.Pointer; +const int __BRIDGEOS_6_5 = 60500; -enum ggml_opt_loss_type { - GGML_OPT_LOSS_TYPE_MEAN(0), - GGML_OPT_LOSS_TYPE_SUM(1), - GGML_OPT_LOSS_TYPE_CROSS_ENTROPY(2), - GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR(3); +const int __BRIDGEOS_6_6 = 60600; - final int value; - const ggml_opt_loss_type(this.value); +const int __BRIDGEOS_7_0 = 70000; - static ggml_opt_loss_type fromValue(int value) => switch (value) { - 0 => GGML_OPT_LOSS_TYPE_MEAN, - 1 => GGML_OPT_LOSS_TYPE_SUM, - 2 => GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, - 3 => GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR, - _ => - throw ArgumentError('Unknown value for ggml_opt_loss_type: $value'), - }; -} +const int __BRIDGEOS_7_1 = 70100; -enum ggml_opt_build_type { - GGML_OPT_BUILD_TYPE_FORWARD(10), - GGML_OPT_BUILD_TYPE_GRAD(20), - GGML_OPT_BUILD_TYPE_OPT(30); +const int __BRIDGEOS_7_2 = 70200; - final int value; - const ggml_opt_build_type(this.value); +const int __BRIDGEOS_7_3 = 70300; - static ggml_opt_build_type fromValue(int value) => switch (value) { - 10 => GGML_OPT_BUILD_TYPE_FORWARD, - 20 => GGML_OPT_BUILD_TYPE_GRAD, - 30 => GGML_OPT_BUILD_TYPE_OPT, - _ => - throw ArgumentError('Unknown value for ggml_opt_build_type: $value'), - }; -} +const int __BRIDGEOS_7_4 = 70400; -enum ggml_opt_optimizer_type { - GGML_OPT_OPTIMIZER_TYPE_ADAMW(0), - GGML_OPT_OPTIMIZER_TYPE_SGD(1), - GGML_OPT_OPTIMIZER_TYPE_COUNT(2); +const int __BRIDGEOS_7_6 = 70600; - final int value; - const ggml_opt_optimizer_type(this.value); +const int __BRIDGEOS_8_0 = 80000; - static ggml_opt_optimizer_type fromValue(int value) => switch (value) { - 0 => GGML_OPT_OPTIMIZER_TYPE_ADAMW, - 1 => GGML_OPT_OPTIMIZER_TYPE_SGD, - 2 => GGML_OPT_OPTIMIZER_TYPE_COUNT, - _ => throw ArgumentError( - 'Unknown value for ggml_opt_optimizer_type: $value'), - }; -} +const int __BRIDGEOS_8_1 = 80100; + +const int __BRIDGEOS_8_2 = 80200; + +const int __BRIDGEOS_8_3 = 80300; + +const int __BRIDGEOS_8_4 = 80400; + +const int __BRIDGEOS_8_5 = 80500; + +const int __BRIDGEOS_8_6 = 80600; + +const int __BRIDGEOS_9_0 = 90000; + +const int __BRIDGEOS_9_1 = 90100; + +const int __BRIDGEOS_9_2 = 90200; + +const int __BRIDGEOS_9_3 = 90300; + +const int __BRIDGEOS_9_4 = 90400; + +const int __BRIDGEOS_9_5 = 90500; -final class UnnamedStruct extends ffi.Struct { - @ffi.Float() - external double alpha; +const int __BRIDGEOS_9_6 = 90600; - @ffi.Float() - external double beta1; +const int __DARWIN_64_BIT_INO_T = 1; - @ffi.Float() - external double beta2; +const int __DARWIN_C_ANSI = 4096; - @ffi.Float() - external double eps; +const int __DARWIN_C_FULL = 900000; - @ffi.Float() - external double wd; -} +const int __DARWIN_C_LEVEL = 900000; -final class UnnamedStruct$1 extends ffi.Struct { - @ffi.Float() - external double alpha; +const int __DARWIN_NON_CANCELABLE = 0; - @ffi.Float() - external double wd; -} +const int __DARWIN_NO_LONG_LONG = 0; -final class ggml_opt_optimizer_params extends ffi.Struct { - external UnnamedStruct adamw; +const int __DARWIN_NULL = 0; - external UnnamedStruct$1 sgd; -} +const int __DARWIN_ONLY_64_BIT_INO_T = 1; -typedef ggml_opt_get_optimizer_paramsFunction = ggml_opt_optimizer_params - Function(ffi.Pointer userdata); -typedef ggml_opt_get_optimizer_params - = ffi.Pointer>; +const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; -final class ggml_opt_params extends ffi.Struct { - external ggml_backend_sched_t backend_sched; +const int __DARWIN_ONLY_VERS_1050 = 1; - external ffi.Pointer ctx_compute; +const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; - external ffi.Pointer inputs; +const int __DARWIN_UNIX03 = 1; - external ffi.Pointer outputs; +const int __DARWIN_VERS_1050 = 1; - @ffi.UnsignedInt() - external int loss_typeAsInt; +const int __DARWIN_WCHAR_MAX = 2147483647; - ggml_opt_loss_type get loss_type => - ggml_opt_loss_type.fromValue(loss_typeAsInt); +const int __DARWIN_WCHAR_MIN = -2147483648; - @ffi.UnsignedInt() - external int build_typeAsInt; +const int __DARWIN_WEOF = -1; - ggml_opt_build_type get build_type => - ggml_opt_build_type.fromValue(build_typeAsInt); +const int __DRIVERKIT_19_0 = 190000; - @ffi.Int32() - external int opt_period; +const int __DRIVERKIT_20_0 = 200000; - external ggml_opt_get_optimizer_params get_opt_pars; +const int __DRIVERKIT_21_0 = 210000; - external ffi.Pointer get_opt_pars_ud; +const int __DRIVERKIT_22_0 = 220000; - @ffi.UnsignedInt() - external int optimizerAsInt; +const int __DRIVERKIT_22_4 = 220400; - ggml_opt_optimizer_type get optimizer => - ggml_opt_optimizer_type.fromValue(optimizerAsInt); -} +const int __DRIVERKIT_22_5 = 220500; -typedef ggml_opt_epoch_callbackFunction = ffi.Void Function( - ffi.Bool train, - ggml_opt_context_t opt_ctx, - ggml_opt_dataset_t dataset, - ggml_opt_result_t result, - ffi.Int64 ibatch, - ffi.Int64 ibatch_max, - ffi.Int64 t_start_us); -typedef Dartggml_opt_epoch_callbackFunction = void Function( - bool train, - ggml_opt_context_t opt_ctx, - ggml_opt_dataset_t dataset, - ggml_opt_result_t result, - int ibatch, - int ibatch_max, - int t_start_us); -typedef ggml_opt_epoch_callback - = ffi.Pointer>; +const int __DRIVERKIT_22_6 = 220600; -enum gguf_type { - GGUF_TYPE_UINT8(0), - GGUF_TYPE_INT8(1), - GGUF_TYPE_UINT16(2), - GGUF_TYPE_INT16(3), - GGUF_TYPE_UINT32(4), - GGUF_TYPE_INT32(5), - GGUF_TYPE_FLOAT32(6), - GGUF_TYPE_BOOL(7), - GGUF_TYPE_STRING(8), - GGUF_TYPE_ARRAY(9), - GGUF_TYPE_UINT64(10), - GGUF_TYPE_INT64(11), - GGUF_TYPE_FLOAT64(12), - GGUF_TYPE_COUNT(13); +const int __DRIVERKIT_23_0 = 230000; - final int value; - const gguf_type(this.value); +const int __DRIVERKIT_23_1 = 230100; - static gguf_type fromValue(int value) => switch (value) { - 0 => GGUF_TYPE_UINT8, - 1 => GGUF_TYPE_INT8, - 2 => GGUF_TYPE_UINT16, - 3 => GGUF_TYPE_INT16, - 4 => GGUF_TYPE_UINT32, - 5 => GGUF_TYPE_INT32, - 6 => GGUF_TYPE_FLOAT32, - 7 => GGUF_TYPE_BOOL, - 8 => GGUF_TYPE_STRING, - 9 => GGUF_TYPE_ARRAY, - 10 => GGUF_TYPE_UINT64, - 11 => GGUF_TYPE_INT64, - 12 => GGUF_TYPE_FLOAT64, - 13 => GGUF_TYPE_COUNT, - _ => throw ArgumentError('Unknown value for gguf_type: $value'), - }; -} +const int __DRIVERKIT_23_2 = 230200; -final class gguf_context extends ffi.Opaque {} +const int __DRIVERKIT_23_3 = 230300; -final class gguf_init_params extends ffi.Struct { - @ffi.Bool() - external bool no_alloc; +const int __DRIVERKIT_23_4 = 230400; - external ffi.Pointer> ctx; -} +const int __DRIVERKIT_23_5 = 230500; -typedef gguf_reader_callback_tFunction = ffi.Size Function( - ffi.Pointer userdata, - ffi.Pointer output, - ffi.Uint64 offset, - ffi.Size len); -typedef Dartgguf_reader_callback_tFunction = int Function( - ffi.Pointer userdata, - ffi.Pointer output, - int offset, - int len); -typedef gguf_reader_callback_t - = ffi.Pointer>; +const int __DRIVERKIT_23_6 = 230600; -final class llama_vocab extends ffi.Opaque {} +const int __DRIVERKIT_24_0 = 240000; -final class llama_model extends ffi.Opaque {} +const int __DRIVERKIT_24_1 = 240100; -final class llama_context extends ffi.Opaque {} +const int __DRIVERKIT_24_2 = 240200; -typedef llama_token = ffi.Int32; -typedef Dartllama_token = int; +const int __DRIVERKIT_24_3 = 240300; -final class llama_token_data extends ffi.Struct { - @llama_token() - external int id; +const int __DRIVERKIT_24_4 = 240400; - @ffi.Float() - external double logit; +const int __DRIVERKIT_24_5 = 240500; - @ffi.Float() - external double p; -} +const int __DRIVERKIT_24_6 = 240600; -final class llama_token_data_array extends ffi.Struct { - external ffi.Pointer data; +const int __DRIVERKIT_25_0 = 250000; - @ffi.Size() - external int size; +const int __DRIVERKIT_25_1 = 250100; - @ffi.Int64() - external int selected; +const int __DRIVERKIT_25_2 = 250200; - @ffi.Bool() - external bool sorted; -} +const int __DRIVERKIT_25_3 = 250300; -final class llama_sampler_data extends ffi.Struct { - external ffi.Pointer logits; +const int __DRIVERKIT_25_4 = 250400; - external ffi.Pointer probs; +const int __DRIVERKIT_25_5 = 250500; - external ffi.Pointer sampled; +const int __ENABLE_LEGACY_MAC_AVAILABILITY = 1; - external ffi.Pointer candidates; -} +const int __IPHONE_10_0 = 100000; -final class llama_sampler_i extends ffi.Struct { - external ffi.Pointer< - ffi.NativeFunction< - ffi.Pointer Function(ffi.Pointer smpl)>> - name; +const int __IPHONE_10_1 = 100100; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer smpl, llama_token token)>> accept; +const int __IPHONE_10_2 = 100200; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Void Function(ffi.Pointer smpl, - ffi.Pointer cur_p)>> apply; +const int __IPHONE_10_3 = 100300; - external ffi.Pointer< - ffi - .NativeFunction smpl)>> - reset; +const int __IPHONE_11_0 = 110000; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Pointer Function( - ffi.Pointer smpl)>> clone; +const int __IPHONE_11_1 = 110100; - external ffi.Pointer< - ffi - .NativeFunction smpl)>> free; +const int __IPHONE_11_2 = 110200; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Bool Function(ffi.Pointer smpl, - ggml_backend_buffer_type_t buft)>> backend_init; +const int __IPHONE_11_3 = 110300; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer smpl, - ffi.Pointer ctx, - ffi.Pointer gf, - ffi.Pointer selected_token)>> backend_accept; +const int __IPHONE_11_4 = 110400; - external ffi.Pointer< - ffi.NativeFunction< - ffi.Void Function( - ffi.Pointer smpl, - ffi.Pointer ctx, - ffi.Pointer gf, - ffi.Pointer data)>> backend_apply; +const int __IPHONE_12_0 = 120000; - external ffi.Pointer< - ffi - .NativeFunction smpl)>> - backend_set_input; -} +const int __IPHONE_12_1 = 120100; + +const int __IPHONE_12_2 = 120200; + +const int __IPHONE_12_3 = 120300; + +const int __IPHONE_12_4 = 120400; -typedef llama_sampler_context_t = ffi.Pointer; +const int __IPHONE_13_0 = 130000; -final class llama_sampler extends ffi.Struct { - external ffi.Pointer iface; +const int __IPHONE_13_1 = 130100; - external llama_sampler_context_t ctx; -} +const int __IPHONE_13_2 = 130200; -final class llama_memory_i extends ffi.Opaque {} +const int __IPHONE_13_3 = 130300; -typedef llama_memory_t = ffi.Pointer; -typedef llama_pos = ffi.Int32; -typedef Dartllama_pos = int; -typedef llama_seq_id = ffi.Int32; -typedef Dartllama_seq_id = int; +const int __IPHONE_13_4 = 130400; -enum llama_vocab_type { - LLAMA_VOCAB_TYPE_NONE(0), - LLAMA_VOCAB_TYPE_SPM(1), - LLAMA_VOCAB_TYPE_BPE(2), - LLAMA_VOCAB_TYPE_WPM(3), - LLAMA_VOCAB_TYPE_UGM(4), - LLAMA_VOCAB_TYPE_RWKV(5), - LLAMA_VOCAB_TYPE_PLAMO2(6); +const int __IPHONE_13_5 = 130500; - final int value; - const llama_vocab_type(this.value); +const int __IPHONE_13_6 = 130600; - static llama_vocab_type fromValue(int value) => switch (value) { - 0 => LLAMA_VOCAB_TYPE_NONE, - 1 => LLAMA_VOCAB_TYPE_SPM, - 2 => LLAMA_VOCAB_TYPE_BPE, - 3 => LLAMA_VOCAB_TYPE_WPM, - 4 => LLAMA_VOCAB_TYPE_UGM, - 5 => LLAMA_VOCAB_TYPE_RWKV, - 6 => LLAMA_VOCAB_TYPE_PLAMO2, - _ => throw ArgumentError('Unknown value for llama_vocab_type: $value'), - }; -} +const int __IPHONE_13_7 = 130700; -enum llama_rope_type { - LLAMA_ROPE_TYPE_NONE(-1), - LLAMA_ROPE_TYPE_NORM(0), - LLAMA_ROPE_TYPE_NEOX(2), - LLAMA_ROPE_TYPE_MROPE(8), - LLAMA_ROPE_TYPE_IMROPE(40), - LLAMA_ROPE_TYPE_VISION(24); +const int __IPHONE_14_0 = 140000; - final int value; - const llama_rope_type(this.value); +const int __IPHONE_14_1 = 140100; - static llama_rope_type fromValue(int value) => switch (value) { - -1 => LLAMA_ROPE_TYPE_NONE, - 0 => LLAMA_ROPE_TYPE_NORM, - 2 => LLAMA_ROPE_TYPE_NEOX, - 8 => LLAMA_ROPE_TYPE_MROPE, - 40 => LLAMA_ROPE_TYPE_IMROPE, - 24 => LLAMA_ROPE_TYPE_VISION, - _ => throw ArgumentError('Unknown value for llama_rope_type: $value'), - }; -} +const int __IPHONE_14_2 = 140200; -enum llama_token_type { - LLAMA_TOKEN_TYPE_UNDEFINED(0), - LLAMA_TOKEN_TYPE_NORMAL(1), - LLAMA_TOKEN_TYPE_UNKNOWN(2), - LLAMA_TOKEN_TYPE_CONTROL(3), - LLAMA_TOKEN_TYPE_USER_DEFINED(4), - LLAMA_TOKEN_TYPE_UNUSED(5), - LLAMA_TOKEN_TYPE_BYTE(6); +const int __IPHONE_14_3 = 140300; - final int value; - const llama_token_type(this.value); +const int __IPHONE_14_5 = 140500; - static llama_token_type fromValue(int value) => switch (value) { - 0 => LLAMA_TOKEN_TYPE_UNDEFINED, - 1 => LLAMA_TOKEN_TYPE_NORMAL, - 2 => LLAMA_TOKEN_TYPE_UNKNOWN, - 3 => LLAMA_TOKEN_TYPE_CONTROL, - 4 => LLAMA_TOKEN_TYPE_USER_DEFINED, - 5 => LLAMA_TOKEN_TYPE_UNUSED, - 6 => LLAMA_TOKEN_TYPE_BYTE, - _ => throw ArgumentError('Unknown value for llama_token_type: $value'), - }; -} +const int __IPHONE_14_6 = 140600; -enum llama_token_attr { - LLAMA_TOKEN_ATTR_UNDEFINED(0), - LLAMA_TOKEN_ATTR_UNKNOWN(1), - LLAMA_TOKEN_ATTR_UNUSED(2), - LLAMA_TOKEN_ATTR_NORMAL(4), - LLAMA_TOKEN_ATTR_CONTROL(8), - LLAMA_TOKEN_ATTR_USER_DEFINED(16), - LLAMA_TOKEN_ATTR_BYTE(32), - LLAMA_TOKEN_ATTR_NORMALIZED(64), - LLAMA_TOKEN_ATTR_LSTRIP(128), - LLAMA_TOKEN_ATTR_RSTRIP(256), - LLAMA_TOKEN_ATTR_SINGLE_WORD(512); +const int __IPHONE_14_7 = 140700; - final int value; - const llama_token_attr(this.value); +const int __IPHONE_14_8 = 140800; - static llama_token_attr fromValue(int value) => switch (value) { - 0 => LLAMA_TOKEN_ATTR_UNDEFINED, - 1 => LLAMA_TOKEN_ATTR_UNKNOWN, - 2 => LLAMA_TOKEN_ATTR_UNUSED, - 4 => LLAMA_TOKEN_ATTR_NORMAL, - 8 => LLAMA_TOKEN_ATTR_CONTROL, - 16 => LLAMA_TOKEN_ATTR_USER_DEFINED, - 32 => LLAMA_TOKEN_ATTR_BYTE, - 64 => LLAMA_TOKEN_ATTR_NORMALIZED, - 128 => LLAMA_TOKEN_ATTR_LSTRIP, - 256 => LLAMA_TOKEN_ATTR_RSTRIP, - 512 => LLAMA_TOKEN_ATTR_SINGLE_WORD, - _ => throw ArgumentError('Unknown value for llama_token_attr: $value'), - }; -} +const int __IPHONE_15_0 = 150000; -enum llama_ftype { - LLAMA_FTYPE_ALL_F32(0), - LLAMA_FTYPE_MOSTLY_F16(1), - LLAMA_FTYPE_MOSTLY_Q4_0(2), - LLAMA_FTYPE_MOSTLY_Q4_1(3), - LLAMA_FTYPE_MOSTLY_Q8_0(7), - LLAMA_FTYPE_MOSTLY_Q5_0(8), - LLAMA_FTYPE_MOSTLY_Q5_1(9), - LLAMA_FTYPE_MOSTLY_Q2_K(10), - LLAMA_FTYPE_MOSTLY_Q3_K_S(11), - LLAMA_FTYPE_MOSTLY_Q3_K_M(12), - LLAMA_FTYPE_MOSTLY_Q3_K_L(13), - LLAMA_FTYPE_MOSTLY_Q4_K_S(14), - LLAMA_FTYPE_MOSTLY_Q4_K_M(15), - LLAMA_FTYPE_MOSTLY_Q5_K_S(16), - LLAMA_FTYPE_MOSTLY_Q5_K_M(17), - LLAMA_FTYPE_MOSTLY_Q6_K(18), - LLAMA_FTYPE_MOSTLY_IQ2_XXS(19), - LLAMA_FTYPE_MOSTLY_IQ2_XS(20), - LLAMA_FTYPE_MOSTLY_Q2_K_S(21), - LLAMA_FTYPE_MOSTLY_IQ3_XS(22), - LLAMA_FTYPE_MOSTLY_IQ3_XXS(23), - LLAMA_FTYPE_MOSTLY_IQ1_S(24), - LLAMA_FTYPE_MOSTLY_IQ4_NL(25), - LLAMA_FTYPE_MOSTLY_IQ3_S(26), - LLAMA_FTYPE_MOSTLY_IQ3_M(27), - LLAMA_FTYPE_MOSTLY_IQ2_S(28), - LLAMA_FTYPE_MOSTLY_IQ2_M(29), - LLAMA_FTYPE_MOSTLY_IQ4_XS(30), - LLAMA_FTYPE_MOSTLY_IQ1_M(31), - LLAMA_FTYPE_MOSTLY_BF16(32), - LLAMA_FTYPE_MOSTLY_TQ1_0(36), - LLAMA_FTYPE_MOSTLY_TQ2_0(37), - LLAMA_FTYPE_MOSTLY_MXFP4_MOE(38), - LLAMA_FTYPE_MOSTLY_NVFP4(39), - LLAMA_FTYPE_MOSTLY_Q1_0(40), - LLAMA_FTYPE_GUESSED(1024); +const int __IPHONE_15_1 = 150100; - final int value; - const llama_ftype(this.value); +const int __IPHONE_15_2 = 150200; - static llama_ftype fromValue(int value) => switch (value) { - 0 => LLAMA_FTYPE_ALL_F32, - 1 => LLAMA_FTYPE_MOSTLY_F16, - 2 => LLAMA_FTYPE_MOSTLY_Q4_0, - 3 => LLAMA_FTYPE_MOSTLY_Q4_1, - 7 => LLAMA_FTYPE_MOSTLY_Q8_0, - 8 => LLAMA_FTYPE_MOSTLY_Q5_0, - 9 => LLAMA_FTYPE_MOSTLY_Q5_1, - 10 => LLAMA_FTYPE_MOSTLY_Q2_K, - 11 => LLAMA_FTYPE_MOSTLY_Q3_K_S, - 12 => LLAMA_FTYPE_MOSTLY_Q3_K_M, - 13 => LLAMA_FTYPE_MOSTLY_Q3_K_L, - 14 => LLAMA_FTYPE_MOSTLY_Q4_K_S, - 15 => LLAMA_FTYPE_MOSTLY_Q4_K_M, - 16 => LLAMA_FTYPE_MOSTLY_Q5_K_S, - 17 => LLAMA_FTYPE_MOSTLY_Q5_K_M, - 18 => LLAMA_FTYPE_MOSTLY_Q6_K, - 19 => LLAMA_FTYPE_MOSTLY_IQ2_XXS, - 20 => LLAMA_FTYPE_MOSTLY_IQ2_XS, - 21 => LLAMA_FTYPE_MOSTLY_Q2_K_S, - 22 => LLAMA_FTYPE_MOSTLY_IQ3_XS, - 23 => LLAMA_FTYPE_MOSTLY_IQ3_XXS, - 24 => LLAMA_FTYPE_MOSTLY_IQ1_S, - 25 => LLAMA_FTYPE_MOSTLY_IQ4_NL, - 26 => LLAMA_FTYPE_MOSTLY_IQ3_S, - 27 => LLAMA_FTYPE_MOSTLY_IQ3_M, - 28 => LLAMA_FTYPE_MOSTLY_IQ2_S, - 29 => LLAMA_FTYPE_MOSTLY_IQ2_M, - 30 => LLAMA_FTYPE_MOSTLY_IQ4_XS, - 31 => LLAMA_FTYPE_MOSTLY_IQ1_M, - 32 => LLAMA_FTYPE_MOSTLY_BF16, - 36 => LLAMA_FTYPE_MOSTLY_TQ1_0, - 37 => LLAMA_FTYPE_MOSTLY_TQ2_0, - 38 => LLAMA_FTYPE_MOSTLY_MXFP4_MOE, - 39 => LLAMA_FTYPE_MOSTLY_NVFP4, - 40 => LLAMA_FTYPE_MOSTLY_Q1_0, - 1024 => LLAMA_FTYPE_GUESSED, - _ => throw ArgumentError('Unknown value for llama_ftype: $value'), - }; -} +const int __IPHONE_15_3 = 150300; -enum llama_rope_scaling_type { - LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED(-1), - LLAMA_ROPE_SCALING_TYPE_NONE(0), - LLAMA_ROPE_SCALING_TYPE_LINEAR(1), - LLAMA_ROPE_SCALING_TYPE_YARN(2), - LLAMA_ROPE_SCALING_TYPE_LONGROPE(3); +const int __IPHONE_15_4 = 150400; - static const LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = - LLAMA_ROPE_SCALING_TYPE_LONGROPE; +const int __IPHONE_15_5 = 150500; - final int value; - const llama_rope_scaling_type(this.value); +const int __IPHONE_15_6 = 150600; - static llama_rope_scaling_type fromValue(int value) => switch (value) { - -1 => LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, - 0 => LLAMA_ROPE_SCALING_TYPE_NONE, - 1 => LLAMA_ROPE_SCALING_TYPE_LINEAR, - 2 => LLAMA_ROPE_SCALING_TYPE_YARN, - 3 => LLAMA_ROPE_SCALING_TYPE_LONGROPE, - _ => throw ArgumentError( - 'Unknown value for llama_rope_scaling_type: $value'), - }; +const int __IPHONE_15_7 = 150700; - @override - String toString() { - if (this == LLAMA_ROPE_SCALING_TYPE_LONGROPE) - return "llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_LONGROPE, llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_MAX_VALUE"; - return super.toString(); - } -} +const int __IPHONE_15_8 = 150800; -enum llama_pooling_type { - LLAMA_POOLING_TYPE_UNSPECIFIED(-1), - LLAMA_POOLING_TYPE_NONE(0), - LLAMA_POOLING_TYPE_MEAN(1), - LLAMA_POOLING_TYPE_CLS(2), - LLAMA_POOLING_TYPE_LAST(3), - LLAMA_POOLING_TYPE_RANK(4); +const int __IPHONE_16_0 = 160000; - final int value; - const llama_pooling_type(this.value); +const int __IPHONE_16_1 = 160100; - static llama_pooling_type fromValue(int value) => switch (value) { - -1 => LLAMA_POOLING_TYPE_UNSPECIFIED, - 0 => LLAMA_POOLING_TYPE_NONE, - 1 => LLAMA_POOLING_TYPE_MEAN, - 2 => LLAMA_POOLING_TYPE_CLS, - 3 => LLAMA_POOLING_TYPE_LAST, - 4 => LLAMA_POOLING_TYPE_RANK, - _ => - throw ArgumentError('Unknown value for llama_pooling_type: $value'), - }; -} +const int __IPHONE_16_2 = 160200; + +const int __IPHONE_16_3 = 160300; -enum llama_attention_type { - LLAMA_ATTENTION_TYPE_UNSPECIFIED(-1), - LLAMA_ATTENTION_TYPE_CAUSAL(0), - LLAMA_ATTENTION_TYPE_NON_CAUSAL(1); +const int __IPHONE_16_4 = 160400; - final int value; - const llama_attention_type(this.value); +const int __IPHONE_16_5 = 160500; - static llama_attention_type fromValue(int value) => switch (value) { - -1 => LLAMA_ATTENTION_TYPE_UNSPECIFIED, - 0 => LLAMA_ATTENTION_TYPE_CAUSAL, - 1 => LLAMA_ATTENTION_TYPE_NON_CAUSAL, - _ => - throw ArgumentError('Unknown value for llama_attention_type: $value'), - }; -} +const int __IPHONE_16_6 = 160600; -enum llama_flash_attn_type { - LLAMA_FLASH_ATTN_TYPE_AUTO(-1), - LLAMA_FLASH_ATTN_TYPE_DISABLED(0), - LLAMA_FLASH_ATTN_TYPE_ENABLED(1); +const int __IPHONE_16_7 = 160700; - final int value; - const llama_flash_attn_type(this.value); +const int __IPHONE_17_0 = 170000; - static llama_flash_attn_type fromValue(int value) => switch (value) { - -1 => LLAMA_FLASH_ATTN_TYPE_AUTO, - 0 => LLAMA_FLASH_ATTN_TYPE_DISABLED, - 1 => LLAMA_FLASH_ATTN_TYPE_ENABLED, - _ => throw ArgumentError( - 'Unknown value for llama_flash_attn_type: $value'), - }; -} +const int __IPHONE_17_1 = 170100; -enum llama_split_mode { - LLAMA_SPLIT_MODE_NONE(0), - LLAMA_SPLIT_MODE_LAYER(1), - LLAMA_SPLIT_MODE_ROW(2), - LLAMA_SPLIT_MODE_TENSOR(3); +const int __IPHONE_17_2 = 170200; - final int value; - const llama_split_mode(this.value); +const int __IPHONE_17_3 = 170300; - static llama_split_mode fromValue(int value) => switch (value) { - 0 => LLAMA_SPLIT_MODE_NONE, - 1 => LLAMA_SPLIT_MODE_LAYER, - 2 => LLAMA_SPLIT_MODE_ROW, - 3 => LLAMA_SPLIT_MODE_TENSOR, - _ => throw ArgumentError('Unknown value for llama_split_mode: $value'), - }; -} +const int __IPHONE_17_4 = 170400; -enum llama_context_type { - LLAMA_CONTEXT_TYPE_DEFAULT(0), - LLAMA_CONTEXT_TYPE_MTP(1); +const int __IPHONE_17_5 = 170500; - final int value; - const llama_context_type(this.value); +const int __IPHONE_17_6 = 170600; - static llama_context_type fromValue(int value) => switch (value) { - 0 => LLAMA_CONTEXT_TYPE_DEFAULT, - 1 => LLAMA_CONTEXT_TYPE_MTP, - _ => - throw ArgumentError('Unknown value for llama_context_type: $value'), - }; -} +const int __IPHONE_17_7 = 170700; -typedef llama_progress_callbackFunction = ffi.Bool Function( - ffi.Float progress, ffi.Pointer user_data); -typedef Dartllama_progress_callbackFunction = bool Function( - double progress, ffi.Pointer user_data); -typedef llama_progress_callback - = ffi.Pointer>; +const int __IPHONE_18_0 = 180000; -final class llama_batch extends ffi.Struct { - @ffi.Int32() - external int n_tokens; +const int __IPHONE_18_1 = 180100; - external ffi.Pointer token; +const int __IPHONE_18_2 = 180200; - external ffi.Pointer embd; +const int __IPHONE_18_3 = 180300; - external ffi.Pointer pos; +const int __IPHONE_18_4 = 180400; - external ffi.Pointer n_seq_id; +const int __IPHONE_18_5 = 180500; - external ffi.Pointer> seq_id; +const int __IPHONE_18_6 = 180600; - external ffi.Pointer logits; -} +const int __IPHONE_19_0 = 190000; -enum llama_model_kv_override_type { - LLAMA_KV_OVERRIDE_TYPE_INT(0), - LLAMA_KV_OVERRIDE_TYPE_FLOAT(1), - LLAMA_KV_OVERRIDE_TYPE_BOOL(2), - LLAMA_KV_OVERRIDE_TYPE_STR(3); +const int __IPHONE_26_0 = 260000; - final int value; - const llama_model_kv_override_type(this.value); +const int __IPHONE_26_1 = 260100; - static llama_model_kv_override_type fromValue(int value) => switch (value) { - 0 => LLAMA_KV_OVERRIDE_TYPE_INT, - 1 => LLAMA_KV_OVERRIDE_TYPE_FLOAT, - 2 => LLAMA_KV_OVERRIDE_TYPE_BOOL, - 3 => LLAMA_KV_OVERRIDE_TYPE_STR, - _ => throw ArgumentError( - 'Unknown value for llama_model_kv_override_type: $value'), - }; -} +const int __IPHONE_26_2 = 260200; -enum llama_model_meta_key { - LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE(0), - LLAMA_MODEL_META_KEY_SAMPLING_TOP_K(1), - LLAMA_MODEL_META_KEY_SAMPLING_TOP_P(2), - LLAMA_MODEL_META_KEY_SAMPLING_MIN_P(3), - LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY(4), - LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD(5), - LLAMA_MODEL_META_KEY_SAMPLING_TEMP(6), - LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N(7), - LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT(8), - LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT(9), - LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU(10), - LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA(11); +const int __IPHONE_26_3 = 260300; - final int value; - const llama_model_meta_key(this.value); +const int __IPHONE_26_4 = 260400; - static llama_model_meta_key fromValue(int value) => switch (value) { - 0 => LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE, - 1 => LLAMA_MODEL_META_KEY_SAMPLING_TOP_K, - 2 => LLAMA_MODEL_META_KEY_SAMPLING_TOP_P, - 3 => LLAMA_MODEL_META_KEY_SAMPLING_MIN_P, - 4 => LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY, - 5 => LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD, - 6 => LLAMA_MODEL_META_KEY_SAMPLING_TEMP, - 7 => LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N, - 8 => LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT, - 9 => LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT, - 10 => LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU, - 11 => LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA, - _ => - throw ArgumentError('Unknown value for llama_model_meta_key: $value'), - }; -} +const int __IPHONE_26_5 = 260500; -final class UnnamedUnion extends ffi.Union { - @ffi.Int64() - external int val_i64; +const int __IPHONE_2_0 = 20000; - @ffi.Double() - external double val_f64; +const int __IPHONE_2_1 = 20100; - @ffi.Bool() - external bool val_bool; +const int __IPHONE_2_2 = 20200; - @ffi.Array.multi([128]) - external ffi.Array val_str; -} +const int __IPHONE_3_0 = 30000; -final class llama_model_kv_override extends ffi.Struct { - @ffi.UnsignedInt() - external int tagAsInt; +const int __IPHONE_3_1 = 30100; - llama_model_kv_override_type get tag => - llama_model_kv_override_type.fromValue(tagAsInt); +const int __IPHONE_3_2 = 30200; - @ffi.Array.multi([128]) - external ffi.Array key; +const int __IPHONE_4_0 = 40000; - external UnnamedUnion unnamed; -} +const int __IPHONE_4_1 = 40100; -final class llama_model_tensor_buft_override extends ffi.Struct { - external ffi.Pointer pattern; +const int __IPHONE_4_2 = 40200; - external ggml_backend_buffer_type_t buft; -} +const int __IPHONE_4_3 = 40300; -final class llama_model_params extends ffi.Struct { - external ffi.Pointer devices; +const int __IPHONE_5_0 = 50000; - external ffi.Pointer tensor_buft_overrides; +const int __IPHONE_5_1 = 50100; - @ffi.Int32() - external int n_gpu_layers; +const int __IPHONE_6_0 = 60000; - @ffi.UnsignedInt() - external int split_modeAsInt; +const int __IPHONE_6_1 = 60100; - llama_split_mode get split_mode => - llama_split_mode.fromValue(split_modeAsInt); +const int __IPHONE_7_0 = 70000; - @ffi.Int32() - external int main_gpu; +const int __IPHONE_7_1 = 70100; - external ffi.Pointer tensor_split; +const int __IPHONE_8_0 = 80000; - external llama_progress_callback progress_callback; +const int __IPHONE_8_1 = 80100; - external ffi.Pointer progress_callback_user_data; +const int __IPHONE_8_2 = 80200; - external ffi.Pointer kv_overrides; +const int __IPHONE_8_3 = 80300; - @ffi.Bool() - external bool vocab_only; +const int __IPHONE_8_4 = 80400; - @ffi.Bool() - external bool use_mmap; +const int __IPHONE_9_0 = 90000; - @ffi.Bool() - external bool use_direct_io; +const int __IPHONE_9_1 = 90100; - @ffi.Bool() - external bool use_mlock; +const int __IPHONE_9_2 = 90200; - @ffi.Bool() - external bool check_tensors; +const int __IPHONE_9_3 = 90300; - @ffi.Bool() - external bool use_extra_bufts; +const int __MAC_10_0 = 1000; - @ffi.Bool() - external bool no_host; +const int __MAC_10_1 = 1010; - @ffi.Bool() - external bool no_alloc; -} +const int __MAC_10_10 = 101000; -final class llama_sampler_seq_config extends ffi.Struct { - @llama_seq_id() - external int seq_id; +const int __MAC_10_10_2 = 101002; - external ffi.Pointer sampler; -} +const int __MAC_10_10_3 = 101003; -final class llama_context_params extends ffi.Struct { - @ffi.Uint32() - external int n_ctx; +const int __MAC_10_11 = 101100; - @ffi.Uint32() - external int n_batch; +const int __MAC_10_11_2 = 101102; - @ffi.Uint32() - external int n_ubatch; +const int __MAC_10_11_3 = 101103; - @ffi.Uint32() - external int n_seq_max; +const int __MAC_10_11_4 = 101104; - @ffi.Uint32() - external int n_rs_seq; +const int __MAC_10_12 = 101200; - @ffi.Uint32() - external int n_outputs_max; +const int __MAC_10_12_1 = 101201; + +const int __MAC_10_12_2 = 101202; + +const int __MAC_10_12_4 = 101204; - @ffi.Int32() - external int n_threads; +const int __MAC_10_13 = 101300; - @ffi.Int32() - external int n_threads_batch; +const int __MAC_10_13_1 = 101301; - @ffi.UnsignedInt() - external int ctx_typeAsInt; +const int __MAC_10_13_2 = 101302; - llama_context_type get ctx_type => - llama_context_type.fromValue(ctx_typeAsInt); +const int __MAC_10_13_4 = 101304; - @ffi.Int() - external int rope_scaling_typeAsInt; +const int __MAC_10_14 = 101400; - llama_rope_scaling_type get rope_scaling_type => - llama_rope_scaling_type.fromValue(rope_scaling_typeAsInt); +const int __MAC_10_14_1 = 101401; - @ffi.Int() - external int pooling_typeAsInt; +const int __MAC_10_14_4 = 101404; - llama_pooling_type get pooling_type => - llama_pooling_type.fromValue(pooling_typeAsInt); +const int __MAC_10_14_5 = 101405; - @ffi.Int() - external int attention_typeAsInt; +const int __MAC_10_14_6 = 101406; - llama_attention_type get attention_type => - llama_attention_type.fromValue(attention_typeAsInt); +const int __MAC_10_15 = 101500; - @ffi.Int() - external int flash_attn_typeAsInt; +const int __MAC_10_15_1 = 101501; - llama_flash_attn_type get flash_attn_type => - llama_flash_attn_type.fromValue(flash_attn_typeAsInt); +const int __MAC_10_15_4 = 101504; - @ffi.Float() - external double rope_freq_base; +const int __MAC_10_16 = 101600; - @ffi.Float() - external double rope_freq_scale; +const int __MAC_10_2 = 1020; - @ffi.Float() - external double yarn_ext_factor; +const int __MAC_10_3 = 1030; - @ffi.Float() - external double yarn_attn_factor; +const int __MAC_10_4 = 1040; - @ffi.Float() - external double yarn_beta_fast; +const int __MAC_10_5 = 1050; - @ffi.Float() - external double yarn_beta_slow; +const int __MAC_10_6 = 1060; - @ffi.Uint32() - external int yarn_orig_ctx; +const int __MAC_10_7 = 1070; - @ffi.Float() - external double defrag_thold; +const int __MAC_10_8 = 1080; - external ggml_backend_sched_eval_callback cb_eval; +const int __MAC_10_9 = 1090; - external ffi.Pointer cb_eval_user_data; +const int __MAC_11_0 = 110000; - @ffi.UnsignedInt() - external int type_kAsInt; +const int __MAC_11_1 = 110100; - ggml_type get type_k => ggml_type.fromValue(type_kAsInt); +const int __MAC_11_3 = 110300; - @ffi.UnsignedInt() - external int type_vAsInt; +const int __MAC_11_4 = 110400; - ggml_type get type_v => ggml_type.fromValue(type_vAsInt); +const int __MAC_11_5 = 110500; - external ggml_abort_callback abort_callback; +const int __MAC_11_6 = 110600; - external ffi.Pointer abort_callback_data; +const int __MAC_12_0 = 120000; - @ffi.Bool() - external bool embeddings; +const int __MAC_12_1 = 120100; - @ffi.Bool() - external bool offload_kqv; +const int __MAC_12_2 = 120200; - @ffi.Bool() - external bool no_perf; +const int __MAC_12_3 = 120300; - @ffi.Bool() - external bool op_offload; +const int __MAC_12_4 = 120400; - @ffi.Bool() - external bool swa_full; +const int __MAC_12_5 = 120500; - @ffi.Bool() - external bool kv_unified; +const int __MAC_12_6 = 120600; - external ffi.Pointer samplers; +const int __MAC_12_7 = 120700; - @ffi.Size() - external int n_samplers; +const int __MAC_13_0 = 130000; - external ffi.Pointer ctx_other; -} +const int __MAC_13_1 = 130100; -final class llama_model_tensor_override extends ffi.Struct { - external ffi.Pointer pattern; +const int __MAC_13_2 = 130200; - @ffi.UnsignedInt() - external int typeAsInt; +const int __MAC_13_3 = 130300; - ggml_type get type => ggml_type.fromValue(typeAsInt); -} +const int __MAC_13_4 = 130400; -final class llama_model_imatrix_data extends ffi.Struct { - external ffi.Pointer name; +const int __MAC_13_5 = 130500; - external ffi.Pointer data; +const int __MAC_13_6 = 130600; - @ffi.Size() - external int size; -} +const int __MAC_13_7 = 130700; -final class llama_model_quantize_params extends ffi.Struct { - @ffi.Int32() - external int nthread; +const int __MAC_14_0 = 140000; - @ffi.UnsignedInt() - external int ftypeAsInt; +const int __MAC_14_1 = 140100; - llama_ftype get ftype => llama_ftype.fromValue(ftypeAsInt); +const int __MAC_14_2 = 140200; - @ffi.UnsignedInt() - external int output_tensor_typeAsInt; +const int __MAC_14_3 = 140300; - ggml_type get output_tensor_type => - ggml_type.fromValue(output_tensor_typeAsInt); +const int __MAC_14_4 = 140400; - @ffi.UnsignedInt() - external int token_embedding_typeAsInt; +const int __MAC_14_5 = 140500; - ggml_type get token_embedding_type => - ggml_type.fromValue(token_embedding_typeAsInt); +const int __MAC_14_6 = 140600; - @ffi.Bool() - external bool allow_requantize; +const int __MAC_14_7 = 140700; - @ffi.Bool() - external bool quantize_output_tensor; +const int __MAC_15_0 = 150000; - @ffi.Bool() - external bool only_copy; +const int __MAC_15_1 = 150100; - @ffi.Bool() - external bool pure; +const int __MAC_15_2 = 150200; - @ffi.Bool() - external bool keep_split; +const int __MAC_15_3 = 150300; - @ffi.Bool() - external bool dry_run; +const int __MAC_15_4 = 150400; - external ffi.Pointer imatrix; +const int __MAC_15_5 = 150500; - external ffi.Pointer kv_overrides; +const int __MAC_15_6 = 150600; - external ffi.Pointer tt_overrides; +const int __MAC_16_0 = 160000; - external ffi.Pointer prune_layers; -} +const int __MAC_26_0 = 260000; -final class llama_logit_bias extends ffi.Struct { - @llama_token() - external int token; +const int __MAC_26_1 = 260100; - @ffi.Float() - external double bias; -} +const int __MAC_26_2 = 260200; -final class llama_sampler_chain_params extends ffi.Struct { - @ffi.Bool() - external bool no_perf; -} +const int __MAC_26_3 = 260300; -final class llama_chat_message extends ffi.Struct { - external ffi.Pointer role; +const int __MAC_26_4 = 260400; - external ffi.Pointer content; -} +const int __MAC_26_5 = 260500; -final class llama_adapter_lora extends ffi.Opaque {} +const int __MAC_OS_X_VERSION_MAX_ALLOWED = 260500; -typedef llama_model_set_tensor_data_tFunction = ffi.Void Function( - ffi.Pointer tensor, ffi.Pointer userdata); -typedef Dartllama_model_set_tensor_data_tFunction = void Function( - ffi.Pointer tensor, ffi.Pointer userdata); -typedef llama_model_set_tensor_data_t - = ffi.Pointer>; -typedef llama_state_seq_flags = ffi.Uint32; -typedef Dartllama_state_seq_flags = int; +const int __MAC_OS_X_VERSION_MIN_REQUIRED = 260000; -final class llama_perf_context_data extends ffi.Struct { - @ffi.Double() - external double t_start_ms; +const int __PTHREAD_ATTR_SIZE__ = 56; - @ffi.Double() - external double t_load_ms; +const int __PTHREAD_CONDATTR_SIZE__ = 8; - @ffi.Double() - external double t_p_eval_ms; +const int __PTHREAD_COND_SIZE__ = 40; - @ffi.Double() - external double t_eval_ms; +const int __PTHREAD_MUTEXATTR_SIZE__ = 8; - @ffi.Int32() - external int n_p_eval; +const int __PTHREAD_MUTEX_SIZE__ = 56; - @ffi.Int32() - external int n_eval; +const int __PTHREAD_ONCE_SIZE__ = 8; - @ffi.Int32() - external int n_reused; -} +const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; -final class llama_perf_sampler_data extends ffi.Struct { - @ffi.Double() - external double t_sample_ms; +const int __PTHREAD_RWLOCK_SIZE__ = 192; - @ffi.Int32() - external int n_sample; -} +const int __PTHREAD_SIZE__ = 8176; -typedef llama_opt_param_filterFunction = ffi.Bool Function( - ffi.Pointer tensor, ffi.Pointer userdata); -typedef Dartllama_opt_param_filterFunction = bool Function( - ffi.Pointer tensor, ffi.Pointer userdata); -typedef llama_opt_param_filter - = ffi.Pointer>; +const int __SALC = 16384; -final class llama_opt_params extends ffi.Struct { - @ffi.Uint32() - external int n_ctx_train; +const int __SAPP = 256; - external llama_opt_param_filter param_filter; +const int __SEOF = 32; - external ffi.Pointer param_filter_ud; +const int __SERR = 64; - external ggml_opt_get_optimizer_params get_opt_pars; +const int __SIGN = 32768; - external ffi.Pointer get_opt_pars_ud; +const int __SLBF = 1; - @ffi.UnsignedInt() - external int optimizer_typeAsInt; +const int __SMBF = 128; - ggml_opt_optimizer_type get optimizer_type => - ggml_opt_optimizer_type.fromValue(optimizer_typeAsInt); -} +const int __SMOD = 8192; -enum mtmd_input_chunk_type { - MTMD_INPUT_CHUNK_TYPE_TEXT(0), - MTMD_INPUT_CHUNK_TYPE_IMAGE(1), - MTMD_INPUT_CHUNK_TYPE_AUDIO(2); +const int __SNBF = 2; - final int value; - const mtmd_input_chunk_type(this.value); +const int __SNPT = 2048; - static mtmd_input_chunk_type fromValue(int value) => switch (value) { - 0 => MTMD_INPUT_CHUNK_TYPE_TEXT, - 1 => MTMD_INPUT_CHUNK_TYPE_IMAGE, - 2 => MTMD_INPUT_CHUNK_TYPE_AUDIO, - _ => throw ArgumentError( - 'Unknown value for mtmd_input_chunk_type: $value'), - }; -} +const int __SOFF = 4096; -final class mtmd_context extends ffi.Opaque {} +const int __SOPT = 1024; -final class mtmd_bitmap extends ffi.Opaque {} +const int __SRD = 4; -final class mtmd_image_tokens extends ffi.Opaque {} +const int __SRW = 16; -final class mtmd_input_chunk extends ffi.Opaque {} +const int __SSTR = 512; -final class mtmd_input_chunks extends ffi.Opaque {} +const int __STDC_WANT_LIB_EXT1__ = 1; -final class mtmd_input_text extends ffi.Struct { - external ffi.Pointer text; +const int __SWR = 8; - @ffi.Bool() - external bool add_special; +const int __TVOS_10_0 = 100000; - @ffi.Bool() - external bool parse_special; -} +const int __TVOS_10_0_1 = 100001; -final class mtmd_context_params extends ffi.Struct { - @ffi.Bool() - external bool use_gpu; +const int __TVOS_10_1 = 100100; - @ffi.Bool() - external bool print_timings; +const int __TVOS_10_2 = 100200; - @ffi.Int() - external int n_threads; +const int __TVOS_11_0 = 110000; - external ffi.Pointer image_marker; +const int __TVOS_11_1 = 110100; - external ffi.Pointer media_marker; +const int __TVOS_11_2 = 110200; - @ffi.Int() - external int flash_attn_typeAsInt; +const int __TVOS_11_3 = 110300; - llama_flash_attn_type get flash_attn_type => - llama_flash_attn_type.fromValue(flash_attn_typeAsInt); +const int __TVOS_11_4 = 110400; - @ffi.Bool() - external bool warmup; +const int __TVOS_12_0 = 120000; - @ffi.Int() - external int image_min_tokens; +const int __TVOS_12_1 = 120100; - @ffi.Int() - external int image_max_tokens; +const int __TVOS_12_2 = 120200; - external ggml_backend_sched_eval_callback cb_eval; +const int __TVOS_12_3 = 120300; - external ffi.Pointer cb_eval_user_data; -} +const int __TVOS_12_4 = 120400; -typedef mtmd_bitmap_lazy_callbackFunction = ffi.Int Function( - ffi.Size chunk_idx, - ffi.Pointer user_data, - ffi.Pointer> out_bitmap, - ffi.Pointer> out_text); -typedef Dartmtmd_bitmap_lazy_callbackFunction = int Function( - int chunk_idx, - ffi.Pointer user_data, - ffi.Pointer> out_bitmap, - ffi.Pointer> out_text); -typedef mtmd_bitmap_lazy_callback - = ffi.Pointer>; +const int __TVOS_13_0 = 130000; -final class mtmd_decoder_pos extends ffi.Struct { - @ffi.Uint32() - external int t; +const int __TVOS_13_2 = 130200; - @ffi.Uint32() - external int x; +const int __TVOS_13_3 = 130300; - @ffi.Uint32() - external int y; +const int __TVOS_13_4 = 130400; - @ffi.Uint32() - external int z; -} +const int __TVOS_14_0 = 140000; -final class mtmd_caps extends ffi.Struct { - @ffi.Bool() - external bool inp_vision; +const int __TVOS_14_1 = 140100; - @ffi.Bool() - external bool inp_audio; -} +const int __TVOS_14_2 = 140200; -final class mtmd_helper_video extends ffi.Opaque {} +const int __TVOS_14_3 = 140300; -final class mtmd_helper_bitmap_wrapper extends ffi.Struct { - external ffi.Pointer bitmap; +const int __TVOS_14_5 = 140500; - external ffi.Pointer video_ctx; -} +const int __TVOS_14_6 = 140600; -final class mtmd_helper_video_info extends ffi.Struct { - @ffi.Uint32() - external int width; +const int __TVOS_14_7 = 140700; - @ffi.Uint32() - external int height; +const int __TVOS_15_0 = 150000; - @ffi.Float() - external double fps; +const int __TVOS_15_1 = 150100; - @ffi.Int32() - external int n_frames; -} +const int __TVOS_15_2 = 150200; -final class mtmd_helper_video_init_params extends ffi.Struct { - @ffi.Float() - external double fps_target; +const int __TVOS_15_3 = 150300; - external ffi.Pointer ffmpeg_bin_dir; +const int __TVOS_15_4 = 150400; - @ffi.Int64() - external int timestamp_interval_ms; -} +const int __TVOS_15_5 = 150500; -const int __bool_true_false_are_defined = 1; +const int __TVOS_15_6 = 150600; -const int true$ = 1; +const int __TVOS_16_0 = 160000; -const int false$ = 0; +const int __TVOS_16_1 = 160100; -const int __has_safe_buffers = 1; +const int __TVOS_16_2 = 160200; -const int __DARWIN_ONLY_64_BIT_INO_T = 1; +const int __TVOS_16_3 = 160300; -const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; +const int __TVOS_16_4 = 160400; -const int __DARWIN_ONLY_VERS_1050 = 1; +const int __TVOS_16_5 = 160500; -const int __DARWIN_UNIX03 = 1; +const int __TVOS_16_6 = 160600; -const int __DARWIN_64_BIT_INO_T = 1; +const int __TVOS_17_0 = 170000; -const int __DARWIN_VERS_1050 = 1; +const int __TVOS_17_1 = 170100; -const int __DARWIN_NON_CANCELABLE = 0; +const int __TVOS_17_2 = 170200; -const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; +const int __TVOS_17_3 = 170300; -const int __DARWIN_C_ANSI = 4096; +const int __TVOS_17_4 = 170400; -const int __DARWIN_C_FULL = 900000; +const int __TVOS_17_5 = 170500; -const int __DARWIN_C_LEVEL = 900000; +const int __TVOS_17_6 = 170600; -const int __STDC_WANT_LIB_EXT1__ = 1; +const int __TVOS_18_0 = 180000; -const int __DARWIN_NO_LONG_LONG = 0; +const int __TVOS_18_1 = 180100; -const int _DARWIN_FEATURE_64_BIT_INODE = 1; +const int __TVOS_18_2 = 180200; -const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; +const int __TVOS_18_3 = 180300; -const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; +const int __TVOS_18_4 = 180400; -const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; +const int __TVOS_18_5 = 180500; -const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; +const int __TVOS_18_6 = 180600; -const int __has_ptrcheck = 0; +const int __TVOS_19_0 = 190000; -const int __has_bounds_safety_attributes = 0; +const int __TVOS_26_0 = 260000; -const int __DARWIN_NULL = 0; +const int __TVOS_26_1 = 260100; -const int __PTHREAD_SIZE__ = 8176; +const int __TVOS_26_2 = 260200; -const int __PTHREAD_ATTR_SIZE__ = 56; +const int __TVOS_26_3 = 260300; -const int __PTHREAD_MUTEXATTR_SIZE__ = 8; +const int __TVOS_26_4 = 260400; -const int __PTHREAD_MUTEX_SIZE__ = 56; +const int __TVOS_26_5 = 260500; -const int __PTHREAD_CONDATTR_SIZE__ = 8; +const int __TVOS_9_0 = 90000; -const int __PTHREAD_COND_SIZE__ = 40; +const int __TVOS_9_1 = 90100; -const int __PTHREAD_ONCE_SIZE__ = 8; +const int __TVOS_9_2 = 90200; -const int __PTHREAD_RWLOCK_SIZE__ = 192; +const int __VISIONOS_1_0 = 10000; -const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; +const int __VISIONOS_1_1 = 10100; -const int __DARWIN_WCHAR_MAX = 2147483647; +const int __VISIONOS_1_2 = 10200; -const int __DARWIN_WCHAR_MIN = -2147483648; +const int __VISIONOS_1_3 = 10300; -const int __DARWIN_WEOF = -1; +const int __VISIONOS_26_0 = 260000; -const int _FORTIFY_SOURCE = 2; +const int __VISIONOS_26_1 = 260100; -const int NULL = 0; +const int __VISIONOS_26_2 = 260200; -const int USER_ADDR_NULL = 0; +const int __VISIONOS_26_3 = 260300; -const int __WORDSIZE = 64; +const int __VISIONOS_26_4 = 260400; -const int INT8_MAX = 127; +const int __VISIONOS_26_5 = 260500; -const int INT16_MAX = 32767; +const int __VISIONOS_2_0 = 20000; -const int INT32_MAX = 2147483647; +const int __VISIONOS_2_1 = 20100; -const int INT64_MAX = 9223372036854775807; +const int __VISIONOS_2_2 = 20200; -const int INT8_MIN = -128; +const int __VISIONOS_2_3 = 20300; -const int INT16_MIN = -32768; +const int __VISIONOS_2_4 = 20400; -const int INT32_MIN = -2147483648; +const int __VISIONOS_2_5 = 20500; -const int INT64_MIN = -9223372036854775808; +const int __VISIONOS_2_6 = 20600; -const int UINT8_MAX = 255; +const int __VISIONOS_3_0 = 30000; -const int UINT16_MAX = 65535; +const int __WATCHOS_10_0 = 100000; -const int UINT32_MAX = 4294967295; +const int __WATCHOS_10_1 = 100100; -const int UINT64_MAX = -1; +const int __WATCHOS_10_2 = 100200; -const int INT_LEAST8_MIN = -128; +const int __WATCHOS_10_3 = 100300; -const int INT_LEAST16_MIN = -32768; +const int __WATCHOS_10_4 = 100400; -const int INT_LEAST32_MIN = -2147483648; +const int __WATCHOS_10_5 = 100500; -const int INT_LEAST64_MIN = -9223372036854775808; +const int __WATCHOS_10_6 = 100600; -const int INT_LEAST8_MAX = 127; +const int __WATCHOS_10_7 = 100700; + +const int __WATCHOS_11_0 = 110000; -const int INT_LEAST16_MAX = 32767; +const int __WATCHOS_11_1 = 110100; -const int INT_LEAST32_MAX = 2147483647; +const int __WATCHOS_11_2 = 110200; -const int INT_LEAST64_MAX = 9223372036854775807; +const int __WATCHOS_11_3 = 110300; -const int UINT_LEAST8_MAX = 255; +const int __WATCHOS_11_4 = 110400; -const int UINT_LEAST16_MAX = 65535; +const int __WATCHOS_11_5 = 110500; -const int UINT_LEAST32_MAX = 4294967295; +const int __WATCHOS_11_6 = 110600; -const int UINT_LEAST64_MAX = -1; +const int __WATCHOS_12_0 = 120000; -const int INT_FAST8_MIN = -128; +const int __WATCHOS_1_0 = 10000; -const int INT_FAST16_MIN = -32768; +const int __WATCHOS_26_0 = 260000; -const int INT_FAST32_MIN = -2147483648; +const int __WATCHOS_26_1 = 260100; -const int INT_FAST64_MIN = -9223372036854775808; +const int __WATCHOS_26_2 = 260200; -const int INT_FAST8_MAX = 127; +const int __WATCHOS_26_3 = 260300; -const int INT_FAST16_MAX = 32767; +const int __WATCHOS_26_4 = 260400; -const int INT_FAST32_MAX = 2147483647; +const int __WATCHOS_26_5 = 260500; -const int INT_FAST64_MAX = 9223372036854775807; +const int __WATCHOS_2_0 = 20000; -const int UINT_FAST8_MAX = 255; +const int __WATCHOS_2_1 = 20100; -const int UINT_FAST16_MAX = 65535; +const int __WATCHOS_2_2 = 20200; -const int UINT_FAST32_MAX = 4294967295; +const int __WATCHOS_3_0 = 30000; -const int UINT_FAST64_MAX = -1; +const int __WATCHOS_3_1 = 30100; -const int INTPTR_MAX = 9223372036854775807; +const int __WATCHOS_3_1_1 = 30101; -const int INTPTR_MIN = -9223372036854775808; +const int __WATCHOS_3_2 = 30200; -const int UINTPTR_MAX = -1; +const int __WATCHOS_4_0 = 40000; -const int INTMAX_MAX = 9223372036854775807; +const int __WATCHOS_4_1 = 40100; -const int UINTMAX_MAX = -1; +const int __WATCHOS_4_2 = 40200; -const int INTMAX_MIN = -9223372036854775808; +const int __WATCHOS_4_3 = 40300; -const int PTRDIFF_MIN = -9223372036854775808; +const int __WATCHOS_5_0 = 50000; -const int PTRDIFF_MAX = 9223372036854775807; +const int __WATCHOS_5_1 = 50100; -const int SIZE_MAX = -1; +const int __WATCHOS_5_2 = 50200; -const int RSIZE_MAX = 9223372036854775807; +const int __WATCHOS_5_3 = 50300; -const int WCHAR_MAX = 2147483647; +const int __WATCHOS_6_0 = 60000; -const int WCHAR_MIN = -2147483648; +const int __WATCHOS_6_1 = 60100; -const int WINT_MIN = -2147483648; +const int __WATCHOS_6_2 = 60200; -const int WINT_MAX = 2147483647; +const int __WATCHOS_7_0 = 70000; -const int SIG_ATOMIC_MIN = -2147483648; +const int __WATCHOS_7_1 = 70100; -const int SIG_ATOMIC_MAX = 2147483647; +const int __WATCHOS_7_2 = 70200; -const int __API_TO_BE_DEPRECATED = 100000; +const int __WATCHOS_7_3 = 70300; -const int __API_TO_BE_DEPRECATED_MACOS = 100000; +const int __WATCHOS_7_4 = 70400; -const int __API_TO_BE_DEPRECATED_MACOSAPPLICATIONEXTENSION = 100000; +const int __WATCHOS_7_5 = 70500; -const int __API_TO_BE_DEPRECATED_IOS = 100000; +const int __WATCHOS_7_6 = 70600; -const int __API_TO_BE_DEPRECATED_IOSAPPLICATIONEXTENSION = 100000; +const int __WATCHOS_8_0 = 80000; -const int __API_TO_BE_DEPRECATED_MACCATALYST = 100000; +const int __WATCHOS_8_1 = 80100; -const int __API_TO_BE_DEPRECATED_MACCATALYSTAPPLICATIONEXTENSION = 100000; +const int __WATCHOS_8_3 = 80300; -const int __API_TO_BE_DEPRECATED_WATCHOS = 100000; +const int __WATCHOS_8_4 = 80400; -const int __API_TO_BE_DEPRECATED_WATCHOSAPPLICATIONEXTENSION = 100000; +const int __WATCHOS_8_5 = 80500; -const int __API_TO_BE_DEPRECATED_TVOS = 100000; +const int __WATCHOS_8_6 = 80600; -const int __API_TO_BE_DEPRECATED_TVOSAPPLICATIONEXTENSION = 100000; +const int __WATCHOS_8_7 = 80700; -const int __API_TO_BE_DEPRECATED_DRIVERKIT = 100000; +const int __WATCHOS_8_8 = 80800; -const int __API_TO_BE_DEPRECATED_VISIONOS = 100000; +const int __WATCHOS_9_0 = 90000; -const int __API_TO_BE_DEPRECATED_VISIONOSAPPLICATIONEXTENSION = 100000; +const int __WATCHOS_9_1 = 90100; -const int __API_TO_BE_DEPRECATED_KERNELKIT = 100000; +const int __WATCHOS_9_2 = 90200; -const int __MAC_10_0 = 1000; +const int __WATCHOS_9_3 = 90300; -const int __MAC_10_1 = 1010; +const int __WATCHOS_9_4 = 90400; -const int __MAC_10_2 = 1020; +const int __WATCHOS_9_5 = 90500; -const int __MAC_10_3 = 1030; +const int __WATCHOS_9_6 = 90600; -const int __MAC_10_4 = 1040; +const int __WORDSIZE = 64; -const int __MAC_10_5 = 1050; +const int __bool_true_false_are_defined = 1; -const int __MAC_10_6 = 1060; +typedef __builtin_va_list = ffi.Pointer; +typedef __darwin_blkcnt_t = __int64_t; +typedef __darwin_blksize_t = __int32_t; +typedef __darwin_clock_t = ffi.UnsignedLong; +typedef Dart__darwin_clock_t = int; +typedef __darwin_ct_rune_t = ffi.Int; +typedef Dart__darwin_ct_rune_t = int; +typedef __darwin_dev_t = __int32_t; +typedef __darwin_fsblkcnt_t = ffi.UnsignedInt; +typedef Dart__darwin_fsblkcnt_t = int; +typedef __darwin_fsfilcnt_t = ffi.UnsignedInt; +typedef Dart__darwin_fsfilcnt_t = int; +typedef __darwin_gid_t = __uint32_t; +typedef __darwin_id_t = __uint32_t; +typedef __darwin_ino64_t = __uint64_t; +typedef __darwin_ino_t = __darwin_ino64_t; +typedef __darwin_intptr_t = ffi.Long; +typedef Dart__darwin_intptr_t = int; +typedef __darwin_mach_port_name_t = __darwin_natural_t; +typedef __darwin_mach_port_t = __darwin_mach_port_name_t; +typedef __darwin_mbstate_t = __mbstate_t; +typedef __darwin_mode_t = __uint16_t; +typedef __darwin_natural_t = ffi.UnsignedInt; +typedef Dart__darwin_natural_t = int; +typedef __darwin_nl_item = ffi.Int; +typedef Dart__darwin_nl_item = int; +typedef __darwin_off_t = __int64_t; +typedef __darwin_pid_t = __int32_t; +typedef __darwin_pthread_attr_t = _opaque_pthread_attr_t; +typedef __darwin_pthread_cond_t = _opaque_pthread_cond_t; +typedef __darwin_pthread_condattr_t = _opaque_pthread_condattr_t; -const int __MAC_10_7 = 1070; +final class __darwin_pthread_handler_rec extends ffi.Struct { + external ffi + .Pointer)>> + __routine; -const int __MAC_10_8 = 1080; + external ffi.Pointer __arg; -const int __MAC_10_9 = 1090; + external ffi.Pointer<__darwin_pthread_handler_rec> __next; +} -const int __MAC_10_10 = 101000; +typedef __darwin_pthread_key_t = ffi.UnsignedLong; +typedef Dart__darwin_pthread_key_t = int; +typedef __darwin_pthread_mutex_t = _opaque_pthread_mutex_t; +typedef __darwin_pthread_mutexattr_t = _opaque_pthread_mutexattr_t; +typedef __darwin_pthread_once_t = _opaque_pthread_once_t; +typedef __darwin_pthread_rwlock_t = _opaque_pthread_rwlock_t; +typedef __darwin_pthread_rwlockattr_t = _opaque_pthread_rwlockattr_t; +typedef __darwin_pthread_t = ffi.Pointer<_opaque_pthread_t>; +typedef __darwin_ptrdiff_t = ffi.Long; +typedef Dart__darwin_ptrdiff_t = int; +typedef __darwin_rune_t = __darwin_wchar_t; +typedef __darwin_sigset_t = __uint32_t; +typedef __darwin_size_t = ffi.UnsignedLong; +typedef Dart__darwin_size_t = int; +typedef __darwin_socklen_t = __uint32_t; +typedef __darwin_ssize_t = ffi.Long; +typedef Dart__darwin_ssize_t = int; +typedef __darwin_suseconds_t = __int32_t; +typedef __darwin_time_t = ffi.Long; +typedef Dart__darwin_time_t = int; +typedef __darwin_uid_t = __uint32_t; +typedef __darwin_useconds_t = __uint32_t; +typedef __darwin_va_list = __builtin_va_list; +typedef __darwin_wchar_t = ffi.Int; +typedef Dart__darwin_wchar_t = int; +typedef __darwin_wctrans_t = ffi.Int; +typedef Dart__darwin_wctrans_t = int; +typedef __darwin_wctype_t = __uint32_t; +typedef __darwin_wint_t = ffi.Int; +typedef Dart__darwin_wint_t = int; -const int __MAC_10_10_2 = 101002; +const int __has_bounds_safety_attributes = 0; -const int __MAC_10_10_3 = 101003; +const int __has_ptrcheck = 0; -const int __MAC_10_11 = 101100; +const int __has_safe_buffers = 1; -const int __MAC_10_11_2 = 101102; +typedef __int16_t = ffi.Short; +typedef Dart__int16_t = int; +typedef __int32_t = ffi.Int; +typedef Dart__int32_t = int; +typedef __int64_t = ffi.LongLong; +typedef Dart__int64_t = int; +typedef __int8_t = ffi.SignedChar; +typedef Dart__int8_t = int; -const int __MAC_10_11_3 = 101103; +final class __mbstate_t extends ffi.Union { + @ffi.Array.multi([128]) + external ffi.Array __mbstate8; -const int __MAC_10_11_4 = 101104; + @ffi.LongLong() + external int _mbstateL; +} -const int __MAC_10_12 = 101200; +final class __sFILE extends ffi.Struct { + external ffi.Pointer _p; -const int __MAC_10_12_1 = 101201; + @ffi.Int() + external int _r; -const int __MAC_10_12_2 = 101202; + @ffi.Int() + external int _w; -const int __MAC_10_12_4 = 101204; + @ffi.Short() + external int _flags; -const int __MAC_10_13 = 101300; + @ffi.Short() + external int _file; -const int __MAC_10_13_1 = 101301; + external __sbuf _bf; -const int __MAC_10_13_2 = 101302; + @ffi.Int() + external int _lbfsize; -const int __MAC_10_13_4 = 101304; + external ffi.Pointer _cookie; -const int __MAC_10_14 = 101400; + external ffi + .Pointer)>> + _close; -const int __MAC_10_14_1 = 101401; + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer, ffi.Int)>> _read; -const int __MAC_10_14_4 = 101404; + external ffi.Pointer< + ffi.NativeFunction< + fpos_t Function(ffi.Pointer, fpos_t, ffi.Int)>> _seek; -const int __MAC_10_14_5 = 101405; + external ffi.Pointer< + ffi.NativeFunction< + ffi.Int Function( + ffi.Pointer, ffi.Pointer, ffi.Int)>> _write; -const int __MAC_10_14_6 = 101406; + external __sbuf _ub; -const int __MAC_10_15 = 101500; + external ffi.Pointer<__sFILEX> _extra; -const int __MAC_10_15_1 = 101501; + @ffi.Int() + external int _ur; -const int __MAC_10_15_4 = 101504; + @ffi.Array.multi([3]) + external ffi.Array _ubuf; -const int __MAC_10_16 = 101600; + @ffi.Array.multi([1]) + external ffi.Array _nbuf; -const int __MAC_11_0 = 110000; + external __sbuf _lb; -const int __MAC_11_1 = 110100; + @ffi.Int() + external int _blksize; -const int __MAC_11_3 = 110300; + @fpos_t() + external int _offset; +} -const int __MAC_11_4 = 110400; +final class __sFILEX extends ffi.Opaque {} -const int __MAC_11_5 = 110500; +final class __sbuf extends ffi.Struct { + external ffi.Pointer _base; -const int __MAC_11_6 = 110600; + @ffi.Int() + external int _size; +} -const int __MAC_12_0 = 120000; +typedef __uint16_t = ffi.UnsignedShort; +typedef Dart__uint16_t = int; +typedef __uint32_t = ffi.UnsignedInt; +typedef Dart__uint32_t = int; +typedef __uint64_t = ffi.UnsignedLongLong; +typedef Dart__uint64_t = int; +typedef __uint8_t = ffi.UnsignedChar; +typedef Dart__uint8_t = int; -const int __MAC_12_1 = 120100; +final class _opaque_pthread_attr_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_12_2 = 120200; + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} -const int __MAC_12_3 = 120300; +final class _opaque_pthread_cond_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_12_4 = 120400; + @ffi.Array.multi([40]) + external ffi.Array __opaque; +} -const int __MAC_12_5 = 120500; +final class _opaque_pthread_condattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_12_6 = 120600; + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} -const int __MAC_12_7 = 120700; +final class _opaque_pthread_mutex_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_13_0 = 130000; + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} -const int __MAC_13_1 = 130100; +final class _opaque_pthread_mutexattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_13_2 = 130200; + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} -const int __MAC_13_3 = 130300; +final class _opaque_pthread_once_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_13_4 = 130400; + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} -const int __MAC_13_5 = 130500; +final class _opaque_pthread_rwlock_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_13_6 = 130600; + @ffi.Array.multi([192]) + external ffi.Array __opaque; +} -const int __MAC_13_7 = 130700; +final class _opaque_pthread_rwlockattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_14_0 = 140000; + @ffi.Array.multi([16]) + external ffi.Array __opaque; +} -const int __MAC_14_1 = 140100; +final class _opaque_pthread_t extends ffi.Struct { + @ffi.Long() + external int __sig; -const int __MAC_14_2 = 140200; + external ffi.Pointer<__darwin_pthread_handler_rec> __cleanup_stack; -const int __MAC_14_3 = 140300; + @ffi.Array.multi([8176]) + external ffi.Array __opaque; +} -const int __MAC_14_4 = 140400; +const int false$ = 0; -const int __MAC_14_5 = 140500; +typedef fpos_t = __darwin_off_t; +typedef ggml_abort_callback + = ffi.Pointer>; +typedef ggml_abort_callbackFunction = ffi.Bool Function( + ffi.Pointer data); +typedef Dartggml_abort_callbackFunction = bool Function( + ffi.Pointer data); +typedef ggml_abort_callback_t + = ffi.Pointer>; +typedef ggml_abort_callback_tFunction = ffi.Void Function( + ffi.Pointer error_message); +typedef Dartggml_abort_callback_tFunction = void Function( + ffi.Pointer error_message); -const int __MAC_14_6 = 140600; +final class ggml_backend extends ffi.Opaque {} -const int __MAC_14_7 = 140700; +final class ggml_backend_buffer extends ffi.Opaque {} -const int __MAC_15_0 = 150000; +typedef ggml_backend_buffer_t = ffi.Pointer; -const int __MAC_15_1 = 150100; +final class ggml_backend_buffer_type extends ffi.Opaque {} -const int __MAC_15_2 = 150200; +typedef ggml_backend_buffer_type_t = ffi.Pointer; -const int __MAC_15_3 = 150300; +enum ggml_backend_buffer_usage { + GGML_BACKEND_BUFFER_USAGE_ANY(0), + GGML_BACKEND_BUFFER_USAGE_WEIGHTS(1), + GGML_BACKEND_BUFFER_USAGE_COMPUTE(2); -const int __MAC_15_4 = 150400; + final int value; + const ggml_backend_buffer_usage(this.value); -const int __MAC_15_5 = 150500; + static ggml_backend_buffer_usage fromValue(int value) => switch (value) { + 0 => GGML_BACKEND_BUFFER_USAGE_ANY, + 1 => GGML_BACKEND_BUFFER_USAGE_WEIGHTS, + 2 => GGML_BACKEND_BUFFER_USAGE_COMPUTE, + _ => throw ArgumentError( + 'Unknown value for ggml_backend_buffer_usage: $value'), + }; +} -const int __MAC_15_6 = 150600; +typedef ggml_backend_comm_allreduce_tensor_t = ffi + .Pointer>; +typedef ggml_backend_comm_allreduce_tensor_tFunction = ffi.Bool Function( + ffi.Pointer comm_ctx, + ffi.Pointer> tensors); +typedef Dartggml_backend_comm_allreduce_tensor_tFunction = bool Function( + ffi.Pointer comm_ctx, + ffi.Pointer> tensors); +typedef ggml_backend_comm_free_t + = ffi.Pointer>; +typedef ggml_backend_comm_free_tFunction = ffi.Void Function( + ffi.Pointer comm_ctx); +typedef Dartggml_backend_comm_free_tFunction = void Function( + ffi.Pointer comm_ctx); +typedef ggml_backend_comm_init_t + = ffi.Pointer>; +typedef ggml_backend_comm_init_tFunction = ffi.Pointer Function( + ffi.Pointer backends, ffi.Size n_backends); +typedef Dartggml_backend_comm_init_tFunction = ffi.Pointer Function( + ffi.Pointer backends, int n_backends); -const int __MAC_16_0 = 160000; +final class ggml_backend_dev_caps extends ffi.Struct { + @ffi.Bool() + external bool async; -const int __MAC_26_0 = 260000; + @ffi.Bool() + external bool host_buffer; -const int __MAC_26_1 = 260100; + @ffi.Bool() + external bool buffer_from_host_ptr; -const int __MAC_26_2 = 260200; + @ffi.Bool() + external bool events; -const int __MAC_26_3 = 260300; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required bool async, + required bool host_buffer, + required bool buffer_from_host_ptr, + required bool events, + }) => + $allocator() + ..ref.async = async + ..ref.host_buffer = host_buffer + ..ref.buffer_from_host_ptr = buffer_from_host_ptr + ..ref.events = events; +} -const int __MAC_26_4 = 260400; +typedef ggml_backend_dev_get_extra_bufts_t = ffi + .Pointer>; +typedef ggml_backend_dev_get_extra_bufts_tFunction + = ffi.Pointer Function( + ggml_backend_dev_t device); -const int __MAC_26_5 = 260500; +final class ggml_backend_dev_props extends ffi.Struct { + external ffi.Pointer name; -const int __IPHONE_2_0 = 20000; + external ffi.Pointer description; -const int __IPHONE_2_1 = 20100; + @ffi.Size() + external int memory_free; -const int __IPHONE_2_2 = 20200; + @ffi.Size() + external int memory_total; -const int __IPHONE_3_0 = 30000; + @ffi.UnsignedInt() + external int typeAsInt; -const int __IPHONE_3_1 = 30100; + ggml_backend_dev_type get type => ggml_backend_dev_type.fromValue(typeAsInt); + set type(ggml_backend_dev_type value) => typeAsInt = value.value; -const int __IPHONE_3_2 = 30200; + external ffi.Pointer device_id; -const int __IPHONE_4_0 = 40000; + external ggml_backend_dev_caps caps; +} -const int __IPHONE_4_1 = 40100; +typedef ggml_backend_dev_t = ffi.Pointer; -const int __IPHONE_4_2 = 40200; +enum ggml_backend_dev_type { + GGML_BACKEND_DEVICE_TYPE_CPU(0), + GGML_BACKEND_DEVICE_TYPE_GPU(1), + GGML_BACKEND_DEVICE_TYPE_IGPU(2), + GGML_BACKEND_DEVICE_TYPE_ACCEL(3), + GGML_BACKEND_DEVICE_TYPE_META(4); -const int __IPHONE_4_3 = 40300; + final int value; + const ggml_backend_dev_type(this.value); -const int __IPHONE_5_0 = 50000; + static ggml_backend_dev_type fromValue(int value) => switch (value) { + 0 => GGML_BACKEND_DEVICE_TYPE_CPU, + 1 => GGML_BACKEND_DEVICE_TYPE_GPU, + 2 => GGML_BACKEND_DEVICE_TYPE_IGPU, + 3 => GGML_BACKEND_DEVICE_TYPE_ACCEL, + 4 => GGML_BACKEND_DEVICE_TYPE_META, + _ => throw ArgumentError( + 'Unknown value for ggml_backend_dev_type: $value'), + }; +} -const int __IPHONE_5_1 = 50100; +final class ggml_backend_device extends ffi.Opaque {} -const int __IPHONE_6_0 = 60000; +typedef ggml_backend_eval_callback + = ffi.Pointer>; +typedef ggml_backend_eval_callbackFunction = ffi.Bool Function( + ffi.Int node_index, + ffi.Pointer t1, + ffi.Pointer t2, + ffi.Pointer user_data); +typedef Dartggml_backend_eval_callbackFunction = bool Function( + int node_index, + ffi.Pointer t1, + ffi.Pointer t2, + ffi.Pointer user_data); -const int __IPHONE_6_1 = 60100; +final class ggml_backend_event extends ffi.Opaque {} -const int __IPHONE_7_0 = 70000; +typedef ggml_backend_event_t = ffi.Pointer; -const int __IPHONE_7_1 = 70100; +final class ggml_backend_feature extends ffi.Struct { + external ffi.Pointer name; -const int __IPHONE_8_0 = 80000; + external ffi.Pointer value; -const int __IPHONE_8_1 = 80100; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer name, + required ffi.Pointer value, + }) => + $allocator() + ..ref.name = name + ..ref.value = value; +} -const int __IPHONE_8_2 = 80200; +typedef ggml_backend_get_features_t + = ffi.Pointer>; +typedef ggml_backend_get_features_tFunction = ffi.Pointer + Function(ggml_backend_reg_t reg); -const int __IPHONE_8_3 = 80300; +final class ggml_backend_graph_copy$1 extends ffi.Struct { + external ggml_backend_buffer_t buffer; -const int __IPHONE_8_4 = 80400; + external ffi.Pointer ctx_allocated; -const int __IPHONE_9_0 = 90000; + external ffi.Pointer ctx_unallocated; -const int __IPHONE_9_1 = 90100; + external ffi.Pointer graph; -const int __IPHONE_9_2 = 90200; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ggml_backend_buffer_t buffer, + required ffi.Pointer ctx_allocated, + required ffi.Pointer ctx_unallocated, + required ffi.Pointer graph, + }) => + $allocator() + ..ref.buffer = buffer + ..ref.ctx_allocated = ctx_allocated + ..ref.ctx_unallocated = ctx_unallocated + ..ref.graph = graph; +} -const int __IPHONE_9_3 = 90300; +typedef ggml_backend_graph_plan_t = ffi.Pointer; +typedef ggml_backend_meta_get_split_state_t = ffi + .Pointer>; +typedef ggml_backend_meta_get_split_state_tFunction + = ggml_backend_meta_split_state Function( + ffi.Pointer tensor, ffi.Pointer userdata); -const int __IPHONE_10_0 = 100000; +enum ggml_backend_meta_split_axis { + GGML_BACKEND_SPLIT_AXIS_0(0), + GGML_BACKEND_SPLIT_AXIS_1(1), + GGML_BACKEND_SPLIT_AXIS_2(2), + GGML_BACKEND_SPLIT_AXIS_3(3), + GGML_BACKEND_SPLIT_AXIS_MIRRORED(10), + GGML_BACKEND_SPLIT_AXIS_PARTIAL(11), + GGML_BACKEND_SPLIT_AXIS_NONE(98), + GGML_BACKEND_SPLIT_AXIS_UNKNOWN(99); -const int __IPHONE_10_1 = 100100; + final int value; + const ggml_backend_meta_split_axis(this.value); -const int __IPHONE_10_2 = 100200; + static ggml_backend_meta_split_axis fromValue(int value) => switch (value) { + 0 => GGML_BACKEND_SPLIT_AXIS_0, + 1 => GGML_BACKEND_SPLIT_AXIS_1, + 2 => GGML_BACKEND_SPLIT_AXIS_2, + 3 => GGML_BACKEND_SPLIT_AXIS_3, + 10 => GGML_BACKEND_SPLIT_AXIS_MIRRORED, + 11 => GGML_BACKEND_SPLIT_AXIS_PARTIAL, + 98 => GGML_BACKEND_SPLIT_AXIS_NONE, + 99 => GGML_BACKEND_SPLIT_AXIS_UNKNOWN, + _ => throw ArgumentError( + 'Unknown value for ggml_backend_meta_split_axis: $value'), + }; +} -const int __IPHONE_10_3 = 100300; +final class ggml_backend_meta_split_state extends ffi.Struct { + @ffi.UnsignedInt() + external int axisAsInt; -const int __IPHONE_11_0 = 110000; + ggml_backend_meta_split_axis get axis => + ggml_backend_meta_split_axis.fromValue(axisAsInt); + set axis(ggml_backend_meta_split_axis value) => axisAsInt = value.value; -const int __IPHONE_11_1 = 110100; + @ffi.Array.multi([256]) + external ffi.Array ne; -const int __IPHONE_11_2 = 110200; + @ffi.Array.multi([16]) + external ffi.Array nr; -const int __IPHONE_11_3 = 110300; + @ffi.Uint32() + external int n_segments; +} -const int __IPHONE_11_4 = 110400; +final class ggml_backend_reg extends ffi.Opaque {} -const int __IPHONE_12_0 = 120000; +typedef ggml_backend_reg_t = ffi.Pointer; -const int __IPHONE_12_1 = 120100; +final class ggml_backend_sched extends ffi.Opaque {} -const int __IPHONE_12_2 = 120200; +typedef ggml_backend_sched_eval_callback + = ffi.Pointer>; +typedef ggml_backend_sched_eval_callbackFunction = ffi.Bool Function( + ffi.Pointer t, ffi.Bool ask, ffi.Pointer user_data); +typedef Dartggml_backend_sched_eval_callbackFunction = bool Function( + ffi.Pointer t, bool ask, ffi.Pointer user_data); +typedef ggml_backend_sched_t = ffi.Pointer; +typedef ggml_backend_set_abort_callback_t = ffi + .Pointer>; +typedef ggml_backend_set_abort_callback_tFunction = ffi.Void Function( + ggml_backend_t backend, + ggml_abort_callback abort_callback, + ffi.Pointer abort_callback_data); +typedef Dartggml_backend_set_abort_callback_tFunction = void Function( + ggml_backend_t backend, + ggml_abort_callback abort_callback, + ffi.Pointer abort_callback_data); +typedef ggml_backend_set_n_threads_t + = ffi.Pointer>; +typedef ggml_backend_set_n_threads_tFunction = ffi.Void Function( + ggml_backend_t backend, ffi.Int n_threads); +typedef Dartggml_backend_set_n_threads_tFunction = void Function( + ggml_backend_t backend, int n_threads); +typedef ggml_backend_split_buffer_type_t + = ffi.Pointer>; +typedef ggml_backend_split_buffer_type_tFunction = ggml_backend_buffer_type_t + Function(ffi.Int main_device, ffi.Pointer tensor_split); +typedef Dartggml_backend_split_buffer_type_tFunction + = ggml_backend_buffer_type_t Function( + int main_device, ffi.Pointer tensor_split); +typedef ggml_backend_t = ffi.Pointer; -const int __IPHONE_12_3 = 120300; +final class ggml_bf16_t extends ffi.Struct { + @ffi.Uint16() + external int bits; -const int __IPHONE_12_4 = 120400; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int bits, + }) => + $allocator()..ref.bits = bits; +} -const int __IPHONE_13_0 = 130000; +final class ggml_cgraph extends ffi.Opaque {} -const int __IPHONE_13_1 = 130100; +final class ggml_context extends ffi.Opaque {} -const int __IPHONE_13_2 = 130200; +final class ggml_cplan extends ffi.Struct { + @ffi.Size() + external int work_size; -const int __IPHONE_13_3 = 130300; + external ffi.Pointer work_data; -const int __IPHONE_13_4 = 130400; + @ffi.Int() + external int n_threads; -const int __IPHONE_13_5 = 130500; + external ffi.Pointer threadpool; -const int __IPHONE_13_6 = 130600; + external ggml_abort_callback abort_callback; -const int __IPHONE_13_7 = 130700; + external ffi.Pointer abort_callback_data; -const int __IPHONE_14_0 = 140000; + @ffi.Bool() + external bool use_ref; -const int __IPHONE_14_1 = 140100; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int work_size, + required ffi.Pointer work_data, + required int n_threads, + required ffi.Pointer threadpool, + required ggml_abort_callback abort_callback, + required ffi.Pointer abort_callback_data, + required bool use_ref, + }) => + $allocator() + ..ref.work_size = work_size + ..ref.work_data = work_data + ..ref.n_threads = n_threads + ..ref.threadpool = threadpool + ..ref.abort_callback = abort_callback + ..ref.abort_callback_data = abort_callback_data + ..ref.use_ref = use_ref; +} -const int __IPHONE_14_2 = 140200; +typedef ggml_custom1_op_t + = ffi.Pointer>; +typedef ggml_custom1_op_tFunction = ffi.Void Function( + ffi.Pointer dst, + ffi.Pointer a, + ffi.Int ith, + ffi.Int nth, + ffi.Pointer userdata); +typedef Dartggml_custom1_op_tFunction = void Function( + ffi.Pointer dst, + ffi.Pointer a, + int ith, + int nth, + ffi.Pointer userdata); +typedef ggml_custom2_op_t + = ffi.Pointer>; +typedef ggml_custom2_op_tFunction = ffi.Void Function( + ffi.Pointer dst, + ffi.Pointer a, + ffi.Pointer b, + ffi.Int ith, + ffi.Int nth, + ffi.Pointer userdata); +typedef Dartggml_custom2_op_tFunction = void Function( + ffi.Pointer dst, + ffi.Pointer a, + ffi.Pointer b, + int ith, + int nth, + ffi.Pointer userdata); +typedef ggml_custom3_op_t + = ffi.Pointer>; +typedef ggml_custom3_op_tFunction = ffi.Void Function( + ffi.Pointer dst, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + ffi.Int ith, + ffi.Int nth, + ffi.Pointer userdata); +typedef Dartggml_custom3_op_tFunction = void Function( + ffi.Pointer dst, + ffi.Pointer a, + ffi.Pointer b, + ffi.Pointer c, + int ith, + int nth, + ffi.Pointer userdata); +typedef ggml_custom_op_t + = ffi.Pointer>; +typedef ggml_custom_op_tFunction = ffi.Void Function( + ffi.Pointer dst, + ffi.Int ith, + ffi.Int nth, + ffi.Pointer userdata); +typedef Dartggml_custom_op_tFunction = void Function( + ffi.Pointer dst, + int ith, + int nth, + ffi.Pointer userdata); +typedef ggml_fp16_t = ffi.Uint16; +typedef Dartggml_fp16_t = int; +typedef ggml_from_float_t + = ffi.Pointer>; +typedef ggml_from_float_tFunction = ffi.Void Function( + ffi.Pointer x, ffi.Pointer y, ffi.Int64 k); +typedef Dartggml_from_float_tFunction = void Function( + ffi.Pointer x, ffi.Pointer y, int k); -const int __IPHONE_14_3 = 140300; +enum ggml_ftype { + GGML_FTYPE_UNKNOWN(-1), + GGML_FTYPE_ALL_F32(0), + GGML_FTYPE_MOSTLY_F16(1), + GGML_FTYPE_MOSTLY_Q4_0(2), + GGML_FTYPE_MOSTLY_Q4_1(3), + GGML_FTYPE_MOSTLY_Q4_1_SOME_F16(4), + GGML_FTYPE_MOSTLY_Q8_0(7), + GGML_FTYPE_MOSTLY_Q5_0(8), + GGML_FTYPE_MOSTLY_Q5_1(9), + GGML_FTYPE_MOSTLY_Q2_K(10), + GGML_FTYPE_MOSTLY_Q3_K(11), + GGML_FTYPE_MOSTLY_Q4_K(12), + GGML_FTYPE_MOSTLY_Q5_K(13), + GGML_FTYPE_MOSTLY_Q6_K(14), + GGML_FTYPE_MOSTLY_IQ2_XXS(15), + GGML_FTYPE_MOSTLY_IQ2_XS(16), + GGML_FTYPE_MOSTLY_IQ3_XXS(17), + GGML_FTYPE_MOSTLY_IQ1_S(18), + GGML_FTYPE_MOSTLY_IQ4_NL(19), + GGML_FTYPE_MOSTLY_IQ3_S(20), + GGML_FTYPE_MOSTLY_IQ2_S(21), + GGML_FTYPE_MOSTLY_IQ4_XS(22), + GGML_FTYPE_MOSTLY_IQ1_M(23), + GGML_FTYPE_MOSTLY_BF16(24), + GGML_FTYPE_MOSTLY_MXFP4(25), + GGML_FTYPE_MOSTLY_NVFP4(26), + GGML_FTYPE_MOSTLY_Q1_0(27), + GGML_FTYPE_MOSTLY_Q2_0(28); -const int __IPHONE_14_5 = 140500; + final int value; + const ggml_ftype(this.value); -const int __IPHONE_14_6 = 140600; + static ggml_ftype fromValue(int value) => switch (value) { + -1 => GGML_FTYPE_UNKNOWN, + 0 => GGML_FTYPE_ALL_F32, + 1 => GGML_FTYPE_MOSTLY_F16, + 2 => GGML_FTYPE_MOSTLY_Q4_0, + 3 => GGML_FTYPE_MOSTLY_Q4_1, + 4 => GGML_FTYPE_MOSTLY_Q4_1_SOME_F16, + 7 => GGML_FTYPE_MOSTLY_Q8_0, + 8 => GGML_FTYPE_MOSTLY_Q5_0, + 9 => GGML_FTYPE_MOSTLY_Q5_1, + 10 => GGML_FTYPE_MOSTLY_Q2_K, + 11 => GGML_FTYPE_MOSTLY_Q3_K, + 12 => GGML_FTYPE_MOSTLY_Q4_K, + 13 => GGML_FTYPE_MOSTLY_Q5_K, + 14 => GGML_FTYPE_MOSTLY_Q6_K, + 15 => GGML_FTYPE_MOSTLY_IQ2_XXS, + 16 => GGML_FTYPE_MOSTLY_IQ2_XS, + 17 => GGML_FTYPE_MOSTLY_IQ3_XXS, + 18 => GGML_FTYPE_MOSTLY_IQ1_S, + 19 => GGML_FTYPE_MOSTLY_IQ4_NL, + 20 => GGML_FTYPE_MOSTLY_IQ3_S, + 21 => GGML_FTYPE_MOSTLY_IQ2_S, + 22 => GGML_FTYPE_MOSTLY_IQ4_XS, + 23 => GGML_FTYPE_MOSTLY_IQ1_M, + 24 => GGML_FTYPE_MOSTLY_BF16, + 25 => GGML_FTYPE_MOSTLY_MXFP4, + 26 => GGML_FTYPE_MOSTLY_NVFP4, + 27 => GGML_FTYPE_MOSTLY_Q1_0, + 28 => GGML_FTYPE_MOSTLY_Q2_0, + _ => throw ArgumentError('Unknown value for ggml_ftype: $value'), + }; +} -const int __IPHONE_14_7 = 140700; +final class ggml_gallocr extends ffi.Opaque {} -const int __IPHONE_14_8 = 140800; +typedef ggml_gallocr_t = ffi.Pointer; -const int __IPHONE_15_0 = 150000; +enum ggml_glu_op { + GGML_GLU_OP_REGLU(0), + GGML_GLU_OP_GEGLU(1), + GGML_GLU_OP_SWIGLU(2), + GGML_GLU_OP_SWIGLU_OAI(3), + GGML_GLU_OP_GEGLU_ERF(4), + GGML_GLU_OP_GEGLU_QUICK(5), + GGML_GLU_OP_COUNT(6); -const int __IPHONE_15_1 = 150100; + final int value; + const ggml_glu_op(this.value); -const int __IPHONE_15_2 = 150200; + static ggml_glu_op fromValue(int value) => switch (value) { + 0 => GGML_GLU_OP_REGLU, + 1 => GGML_GLU_OP_GEGLU, + 2 => GGML_GLU_OP_SWIGLU, + 3 => GGML_GLU_OP_SWIGLU_OAI, + 4 => GGML_GLU_OP_GEGLU_ERF, + 5 => GGML_GLU_OP_GEGLU_QUICK, + 6 => GGML_GLU_OP_COUNT, + _ => throw ArgumentError('Unknown value for ggml_glu_op: $value'), + }; +} -const int __IPHONE_15_3 = 150300; +typedef ggml_guid_t = ffi.Pointer>; -const int __IPHONE_15_4 = 150400; +final class ggml_init_params extends ffi.Struct { + @ffi.Size() + external int mem_size; -const int __IPHONE_15_5 = 150500; + external ffi.Pointer mem_buffer; -const int __IPHONE_15_6 = 150600; + @ffi.Bool() + external bool no_alloc; -const int __IPHONE_15_7 = 150700; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int mem_size, + required ffi.Pointer mem_buffer, + required bool no_alloc, + }) => + $allocator() + ..ref.mem_size = mem_size + ..ref.mem_buffer = mem_buffer + ..ref.no_alloc = no_alloc; +} -const int __IPHONE_15_8 = 150800; +typedef ggml_log_callback + = ffi.Pointer>; +typedef ggml_log_callbackFunction = ffi.Void Function(ffi.UnsignedInt level, + ffi.Pointer text, ffi.Pointer user_data); +typedef Dartggml_log_callbackFunction = void Function(ggml_log_level level, + ffi.Pointer text, ffi.Pointer user_data); -const int __IPHONE_16_0 = 160000; +enum ggml_log_level { + GGML_LOG_LEVEL_NONE(0), + GGML_LOG_LEVEL_DEBUG(1), + GGML_LOG_LEVEL_INFO(2), + GGML_LOG_LEVEL_WARN(3), + GGML_LOG_LEVEL_ERROR(4), + GGML_LOG_LEVEL_CONT(5); -const int __IPHONE_16_1 = 160100; + final int value; + const ggml_log_level(this.value); -const int __IPHONE_16_2 = 160200; + static ggml_log_level fromValue(int value) => switch (value) { + 0 => GGML_LOG_LEVEL_NONE, + 1 => GGML_LOG_LEVEL_DEBUG, + 2 => GGML_LOG_LEVEL_INFO, + 3 => GGML_LOG_LEVEL_WARN, + 4 => GGML_LOG_LEVEL_ERROR, + 5 => GGML_LOG_LEVEL_CONT, + _ => throw ArgumentError('Unknown value for ggml_log_level: $value'), + }; +} -const int __IPHONE_16_3 = 160300; +enum ggml_numa_strategy { + GGML_NUMA_STRATEGY_DISABLED(0), + GGML_NUMA_STRATEGY_DISTRIBUTE(1), + GGML_NUMA_STRATEGY_ISOLATE(2), + GGML_NUMA_STRATEGY_NUMACTL(3), + GGML_NUMA_STRATEGY_MIRROR(4), + GGML_NUMA_STRATEGY_COUNT(5); -const int __IPHONE_16_4 = 160400; + final int value; + const ggml_numa_strategy(this.value); -const int __IPHONE_16_5 = 160500; + static ggml_numa_strategy fromValue(int value) => switch (value) { + 0 => GGML_NUMA_STRATEGY_DISABLED, + 1 => GGML_NUMA_STRATEGY_DISTRIBUTE, + 2 => GGML_NUMA_STRATEGY_ISOLATE, + 3 => GGML_NUMA_STRATEGY_NUMACTL, + 4 => GGML_NUMA_STRATEGY_MIRROR, + 5 => GGML_NUMA_STRATEGY_COUNT, + _ => + throw ArgumentError('Unknown value for ggml_numa_strategy: $value'), + }; +} -const int __IPHONE_16_6 = 160600; +final class ggml_object extends ffi.Opaque {} -const int __IPHONE_16_7 = 160700; +enum ggml_object_type { + GGML_OBJECT_TYPE_TENSOR(0), + GGML_OBJECT_TYPE_GRAPH(1), + GGML_OBJECT_TYPE_WORK_BUFFER(2); -const int __IPHONE_17_0 = 170000; + final int value; + const ggml_object_type(this.value); -const int __IPHONE_17_1 = 170100; + static ggml_object_type fromValue(int value) => switch (value) { + 0 => GGML_OBJECT_TYPE_TENSOR, + 1 => GGML_OBJECT_TYPE_GRAPH, + 2 => GGML_OBJECT_TYPE_WORK_BUFFER, + _ => throw ArgumentError('Unknown value for ggml_object_type: $value'), + }; +} -const int __IPHONE_17_2 = 170200; +enum ggml_op { + GGML_OP_NONE(0), + GGML_OP_DUP(1), + GGML_OP_ADD(2), + GGML_OP_ADD_ID(3), + GGML_OP_ADD1(4), + GGML_OP_ACC(5), + GGML_OP_SUB(6), + GGML_OP_MUL(7), + GGML_OP_DIV(8), + GGML_OP_SQR(9), + GGML_OP_SQRT(10), + GGML_OP_LOG(11), + GGML_OP_SIN(12), + GGML_OP_COS(13), + GGML_OP_SUM(14), + GGML_OP_SUM_ROWS(15), + GGML_OP_CUMSUM(16), + GGML_OP_MEAN(17), + GGML_OP_ARGMAX(18), + GGML_OP_COUNT_EQUAL(19), + GGML_OP_REPEAT(20), + GGML_OP_REPEAT_BACK(21), + GGML_OP_CONCAT(22), + GGML_OP_SILU_BACK(23), + GGML_OP_NORM(24), + GGML_OP_RMS_NORM(25), + GGML_OP_RMS_NORM_BACK(26), + GGML_OP_GROUP_NORM(27), + GGML_OP_L2_NORM(28), + GGML_OP_MUL_MAT(29), + GGML_OP_MUL_MAT_ID(30), + GGML_OP_OUT_PROD(31), + GGML_OP_SCALE(32), + GGML_OP_SET(33), + GGML_OP_CPY(34), + GGML_OP_CONT(35), + GGML_OP_RESHAPE(36), + GGML_OP_VIEW(37), + GGML_OP_PERMUTE(38), + GGML_OP_TRANSPOSE(39), + GGML_OP_GET_ROWS(40), + GGML_OP_GET_ROWS_BACK(41), + GGML_OP_SET_ROWS(42), + GGML_OP_DIAG(43), + GGML_OP_DIAG_MASK_INF(44), + GGML_OP_DIAG_MASK_ZERO(45), + GGML_OP_SOFT_MAX(46), + GGML_OP_SOFT_MAX_BACK(47), + GGML_OP_ROPE(48), + GGML_OP_ROPE_BACK(49), + GGML_OP_CLAMP(50), + GGML_OP_CONV_TRANSPOSE_1D(51), + GGML_OP_IM2COL(52), + GGML_OP_IM2COL_BACK(53), + GGML_OP_IM2COL_3D(54), + GGML_OP_COL2IM_1D(55), + GGML_OP_CONV_2D(56), + GGML_OP_CONV_3D(57), + GGML_OP_CONV_2D_DW(58), + GGML_OP_CONV_TRANSPOSE_2D(59), + GGML_OP_POOL_1D(60), + GGML_OP_POOL_2D(61), + GGML_OP_POOL_2D_BACK(62), + GGML_OP_UPSCALE(63), + GGML_OP_PAD(64), + GGML_OP_PAD_REFLECT_1D(65), + GGML_OP_ROLL(66), + GGML_OP_ARANGE(67), + GGML_OP_TIMESTEP_EMBEDDING(68), + GGML_OP_ARGSORT(69), + GGML_OP_TOP_K(70), + GGML_OP_LEAKY_RELU(71), + GGML_OP_TRI(72), + GGML_OP_FILL(73), + GGML_OP_FLASH_ATTN_EXT(74), + GGML_OP_FLASH_ATTN_BACK(75), + GGML_OP_SSM_CONV(76), + GGML_OP_SSM_SCAN(77), + GGML_OP_WIN_PART(78), + GGML_OP_WIN_UNPART(79), + GGML_OP_GET_REL_POS(80), + GGML_OP_ADD_REL_POS(81), + GGML_OP_RWKV_WKV6(82), + GGML_OP_GATED_LINEAR_ATTN(83), + GGML_OP_RWKV_WKV7(84), + GGML_OP_SOLVE_TRI(85), + GGML_OP_GATED_DELTA_NET(86), + GGML_OP_LIGHTNING_INDEXER(87), + GGML_OP_DSV4_HC_COMB(88), + GGML_OP_DSV4_HC_PRE(89), + GGML_OP_DSV4_HC_POST(90), + GGML_OP_UNARY(91), + GGML_OP_MAP_CUSTOM1(92), + GGML_OP_MAP_CUSTOM2(93), + GGML_OP_MAP_CUSTOM3(94), + GGML_OP_CUSTOM(95), + GGML_OP_CROSS_ENTROPY_LOSS(96), + GGML_OP_CROSS_ENTROPY_LOSS_BACK(97), + GGML_OP_OPT_STEP_ADAMW(98), + GGML_OP_OPT_STEP_SGD(99), + GGML_OP_GLU(100), + GGML_OP_COUNT(101); -const int __IPHONE_17_3 = 170300; + final int value; + const ggml_op(this.value); -const int __IPHONE_17_4 = 170400; + static ggml_op fromValue(int value) => switch (value) { + 0 => GGML_OP_NONE, + 1 => GGML_OP_DUP, + 2 => GGML_OP_ADD, + 3 => GGML_OP_ADD_ID, + 4 => GGML_OP_ADD1, + 5 => GGML_OP_ACC, + 6 => GGML_OP_SUB, + 7 => GGML_OP_MUL, + 8 => GGML_OP_DIV, + 9 => GGML_OP_SQR, + 10 => GGML_OP_SQRT, + 11 => GGML_OP_LOG, + 12 => GGML_OP_SIN, + 13 => GGML_OP_COS, + 14 => GGML_OP_SUM, + 15 => GGML_OP_SUM_ROWS, + 16 => GGML_OP_CUMSUM, + 17 => GGML_OP_MEAN, + 18 => GGML_OP_ARGMAX, + 19 => GGML_OP_COUNT_EQUAL, + 20 => GGML_OP_REPEAT, + 21 => GGML_OP_REPEAT_BACK, + 22 => GGML_OP_CONCAT, + 23 => GGML_OP_SILU_BACK, + 24 => GGML_OP_NORM, + 25 => GGML_OP_RMS_NORM, + 26 => GGML_OP_RMS_NORM_BACK, + 27 => GGML_OP_GROUP_NORM, + 28 => GGML_OP_L2_NORM, + 29 => GGML_OP_MUL_MAT, + 30 => GGML_OP_MUL_MAT_ID, + 31 => GGML_OP_OUT_PROD, + 32 => GGML_OP_SCALE, + 33 => GGML_OP_SET, + 34 => GGML_OP_CPY, + 35 => GGML_OP_CONT, + 36 => GGML_OP_RESHAPE, + 37 => GGML_OP_VIEW, + 38 => GGML_OP_PERMUTE, + 39 => GGML_OP_TRANSPOSE, + 40 => GGML_OP_GET_ROWS, + 41 => GGML_OP_GET_ROWS_BACK, + 42 => GGML_OP_SET_ROWS, + 43 => GGML_OP_DIAG, + 44 => GGML_OP_DIAG_MASK_INF, + 45 => GGML_OP_DIAG_MASK_ZERO, + 46 => GGML_OP_SOFT_MAX, + 47 => GGML_OP_SOFT_MAX_BACK, + 48 => GGML_OP_ROPE, + 49 => GGML_OP_ROPE_BACK, + 50 => GGML_OP_CLAMP, + 51 => GGML_OP_CONV_TRANSPOSE_1D, + 52 => GGML_OP_IM2COL, + 53 => GGML_OP_IM2COL_BACK, + 54 => GGML_OP_IM2COL_3D, + 55 => GGML_OP_COL2IM_1D, + 56 => GGML_OP_CONV_2D, + 57 => GGML_OP_CONV_3D, + 58 => GGML_OP_CONV_2D_DW, + 59 => GGML_OP_CONV_TRANSPOSE_2D, + 60 => GGML_OP_POOL_1D, + 61 => GGML_OP_POOL_2D, + 62 => GGML_OP_POOL_2D_BACK, + 63 => GGML_OP_UPSCALE, + 64 => GGML_OP_PAD, + 65 => GGML_OP_PAD_REFLECT_1D, + 66 => GGML_OP_ROLL, + 67 => GGML_OP_ARANGE, + 68 => GGML_OP_TIMESTEP_EMBEDDING, + 69 => GGML_OP_ARGSORT, + 70 => GGML_OP_TOP_K, + 71 => GGML_OP_LEAKY_RELU, + 72 => GGML_OP_TRI, + 73 => GGML_OP_FILL, + 74 => GGML_OP_FLASH_ATTN_EXT, + 75 => GGML_OP_FLASH_ATTN_BACK, + 76 => GGML_OP_SSM_CONV, + 77 => GGML_OP_SSM_SCAN, + 78 => GGML_OP_WIN_PART, + 79 => GGML_OP_WIN_UNPART, + 80 => GGML_OP_GET_REL_POS, + 81 => GGML_OP_ADD_REL_POS, + 82 => GGML_OP_RWKV_WKV6, + 83 => GGML_OP_GATED_LINEAR_ATTN, + 84 => GGML_OP_RWKV_WKV7, + 85 => GGML_OP_SOLVE_TRI, + 86 => GGML_OP_GATED_DELTA_NET, + 87 => GGML_OP_LIGHTNING_INDEXER, + 88 => GGML_OP_DSV4_HC_COMB, + 89 => GGML_OP_DSV4_HC_PRE, + 90 => GGML_OP_DSV4_HC_POST, + 91 => GGML_OP_UNARY, + 92 => GGML_OP_MAP_CUSTOM1, + 93 => GGML_OP_MAP_CUSTOM2, + 94 => GGML_OP_MAP_CUSTOM3, + 95 => GGML_OP_CUSTOM, + 96 => GGML_OP_CROSS_ENTROPY_LOSS, + 97 => GGML_OP_CROSS_ENTROPY_LOSS_BACK, + 98 => GGML_OP_OPT_STEP_ADAMW, + 99 => GGML_OP_OPT_STEP_SGD, + 100 => GGML_OP_GLU, + 101 => GGML_OP_COUNT, + _ => throw ArgumentError('Unknown value for ggml_op: $value'), + }; +} -const int __IPHONE_17_5 = 170500; +enum ggml_op_hint { + GGML_HINT_NONE(0), + GGML_HINT_SRC0_IS_HADAMARD(1); -const int __IPHONE_17_6 = 170600; + final int value; + const ggml_op_hint(this.value); -const int __IPHONE_17_7 = 170700; + static ggml_op_hint fromValue(int value) => switch (value) { + 0 => GGML_HINT_NONE, + 1 => GGML_HINT_SRC0_IS_HADAMARD, + _ => throw ArgumentError('Unknown value for ggml_op_hint: $value'), + }; +} -const int __IPHONE_18_0 = 180000; +enum ggml_op_pool { + GGML_OP_POOL_MAX(0), + GGML_OP_POOL_AVG(1), + GGML_OP_POOL_COUNT(2); -const int __IPHONE_18_1 = 180100; + final int value; + const ggml_op_pool(this.value); -const int __IPHONE_18_2 = 180200; + static ggml_op_pool fromValue(int value) => switch (value) { + 0 => GGML_OP_POOL_MAX, + 1 => GGML_OP_POOL_AVG, + 2 => GGML_OP_POOL_COUNT, + _ => throw ArgumentError('Unknown value for ggml_op_pool: $value'), + }; +} -const int __IPHONE_18_3 = 180300; +enum ggml_opt_build_type { + GGML_OPT_BUILD_TYPE_FORWARD(10), + GGML_OPT_BUILD_TYPE_GRAD(20), + GGML_OPT_BUILD_TYPE_OPT(30); -const int __IPHONE_18_4 = 180400; + final int value; + const ggml_opt_build_type(this.value); -const int __IPHONE_18_5 = 180500; + static ggml_opt_build_type fromValue(int value) => switch (value) { + 10 => GGML_OPT_BUILD_TYPE_FORWARD, + 20 => GGML_OPT_BUILD_TYPE_GRAD, + 30 => GGML_OPT_BUILD_TYPE_OPT, + _ => + throw ArgumentError('Unknown value for ggml_opt_build_type: $value'), + }; +} -const int __IPHONE_18_6 = 180600; +final class ggml_opt_context extends ffi.Opaque {} -const int __IPHONE_19_0 = 190000; +typedef ggml_opt_context_t = ffi.Pointer; -const int __IPHONE_26_0 = 260000; +final class ggml_opt_dataset extends ffi.Opaque {} -const int __IPHONE_26_1 = 260100; +typedef ggml_opt_dataset_t = ffi.Pointer; +typedef ggml_opt_epoch_callback + = ffi.Pointer>; +typedef ggml_opt_epoch_callbackFunction = ffi.Void Function( + ffi.Bool train, + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, + ffi.Int64 ibatch, + ffi.Int64 ibatch_max, + ffi.Int64 t_start_us); +typedef Dartggml_opt_epoch_callbackFunction = void Function( + bool train, + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, + int ibatch, + int ibatch_max, + int t_start_us); +typedef ggml_opt_get_optimizer_params + = ffi.Pointer>; +typedef ggml_opt_get_optimizer_paramsFunction = ggml_opt_optimizer_params + Function(ffi.Pointer userdata); -const int __IPHONE_26_2 = 260200; +enum ggml_opt_loss_type { + GGML_OPT_LOSS_TYPE_MEAN(0), + GGML_OPT_LOSS_TYPE_SUM(1), + GGML_OPT_LOSS_TYPE_CROSS_ENTROPY(2), + GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR(3); -const int __IPHONE_26_3 = 260300; + final int value; + const ggml_opt_loss_type(this.value); -const int __IPHONE_26_4 = 260400; + static ggml_opt_loss_type fromValue(int value) => switch (value) { + 0 => GGML_OPT_LOSS_TYPE_MEAN, + 1 => GGML_OPT_LOSS_TYPE_SUM, + 2 => GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, + 3 => GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR, + _ => + throw ArgumentError('Unknown value for ggml_opt_loss_type: $value'), + }; +} -const int __IPHONE_26_5 = 260500; +final class ggml_opt_optimizer_params extends ffi.Struct { + external UnnamedStruct adamw; -const int __WATCHOS_1_0 = 10000; + external UnnamedStruct$1 sgd; +} -const int __WATCHOS_2_0 = 20000; +enum ggml_opt_optimizer_type { + GGML_OPT_OPTIMIZER_TYPE_ADAMW(0), + GGML_OPT_OPTIMIZER_TYPE_SGD(1), + GGML_OPT_OPTIMIZER_TYPE_COUNT(2); -const int __WATCHOS_2_1 = 20100; + final int value; + const ggml_opt_optimizer_type(this.value); -const int __WATCHOS_2_2 = 20200; + static ggml_opt_optimizer_type fromValue(int value) => switch (value) { + 0 => GGML_OPT_OPTIMIZER_TYPE_ADAMW, + 1 => GGML_OPT_OPTIMIZER_TYPE_SGD, + 2 => GGML_OPT_OPTIMIZER_TYPE_COUNT, + _ => throw ArgumentError( + 'Unknown value for ggml_opt_optimizer_type: $value'), + }; +} -const int __WATCHOS_3_0 = 30000; +final class ggml_opt_params extends ffi.Struct { + external ggml_backend_sched_t backend_sched; -const int __WATCHOS_3_1 = 30100; + external ffi.Pointer ctx_compute; -const int __WATCHOS_3_1_1 = 30101; + external ffi.Pointer inputs; -const int __WATCHOS_3_2 = 30200; + external ffi.Pointer outputs; -const int __WATCHOS_4_0 = 40000; + @ffi.UnsignedInt() + external int loss_typeAsInt; -const int __WATCHOS_4_1 = 40100; + ggml_opt_loss_type get loss_type => + ggml_opt_loss_type.fromValue(loss_typeAsInt); + set loss_type(ggml_opt_loss_type value) => loss_typeAsInt = value.value; -const int __WATCHOS_4_2 = 40200; + @ffi.UnsignedInt() + external int build_typeAsInt; -const int __WATCHOS_4_3 = 40300; + ggml_opt_build_type get build_type => + ggml_opt_build_type.fromValue(build_typeAsInt); + set build_type(ggml_opt_build_type value) => build_typeAsInt = value.value; -const int __WATCHOS_5_0 = 50000; + @ffi.Int32() + external int opt_period; -const int __WATCHOS_5_1 = 50100; + external ggml_opt_get_optimizer_params get_opt_pars; -const int __WATCHOS_5_2 = 50200; + external ffi.Pointer get_opt_pars_ud; -const int __WATCHOS_5_3 = 50300; + @ffi.UnsignedInt() + external int optimizerAsInt; -const int __WATCHOS_6_0 = 60000; + ggml_opt_optimizer_type get optimizer => + ggml_opt_optimizer_type.fromValue(optimizerAsInt); + set optimizer(ggml_opt_optimizer_type value) => optimizerAsInt = value.value; + + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ggml_backend_sched_t backend_sched, + required ffi.Pointer ctx_compute, + required ffi.Pointer inputs, + required ffi.Pointer outputs, + required ggml_opt_loss_type loss_type, + required ggml_opt_build_type build_type, + required int opt_period, + required ggml_opt_get_optimizer_params get_opt_pars, + required ffi.Pointer get_opt_pars_ud, + required ggml_opt_optimizer_type optimizer, + }) => + $allocator() + ..ref.backend_sched = backend_sched + ..ref.ctx_compute = ctx_compute + ..ref.inputs = inputs + ..ref.outputs = outputs + ..ref.loss_type = loss_type + ..ref.build_type = build_type + ..ref.opt_period = opt_period + ..ref.get_opt_pars = get_opt_pars + ..ref.get_opt_pars_ud = get_opt_pars_ud + ..ref.optimizer = optimizer; +} -const int __WATCHOS_6_1 = 60100; +final class ggml_opt_result extends ffi.Opaque {} -const int __WATCHOS_6_2 = 60200; +typedef ggml_opt_result_t = ffi.Pointer; -const int __WATCHOS_7_0 = 70000; +enum ggml_prec { + GGML_PREC_DEFAULT(0), + GGML_PREC_F32(10); -const int __WATCHOS_7_1 = 70100; + final int value; + const ggml_prec(this.value); -const int __WATCHOS_7_2 = 70200; + static ggml_prec fromValue(int value) => switch (value) { + 0 => GGML_PREC_DEFAULT, + 10 => GGML_PREC_F32, + _ => throw ArgumentError('Unknown value for ggml_prec: $value'), + }; +} -const int __WATCHOS_7_3 = 70300; +enum ggml_scale_flag { + GGML_SCALE_FLAG_ALIGN_CORNERS(256), + GGML_SCALE_FLAG_ANTIALIAS(512); -const int __WATCHOS_7_4 = 70400; + final int value; + const ggml_scale_flag(this.value); -const int __WATCHOS_7_5 = 70500; + static ggml_scale_flag fromValue(int value) => switch (value) { + 256 => GGML_SCALE_FLAG_ALIGN_CORNERS, + 512 => GGML_SCALE_FLAG_ANTIALIAS, + _ => throw ArgumentError('Unknown value for ggml_scale_flag: $value'), + }; +} -const int __WATCHOS_7_6 = 70600; +enum ggml_scale_mode { + GGML_SCALE_MODE_NEAREST(0), + GGML_SCALE_MODE_BILINEAR(1), + GGML_SCALE_MODE_BICUBIC(2), + GGML_SCALE_MODE_COUNT(3); -const int __WATCHOS_8_0 = 80000; + final int value; + const ggml_scale_mode(this.value); -const int __WATCHOS_8_1 = 80100; + static ggml_scale_mode fromValue(int value) => switch (value) { + 0 => GGML_SCALE_MODE_NEAREST, + 1 => GGML_SCALE_MODE_BILINEAR, + 2 => GGML_SCALE_MODE_BICUBIC, + 3 => GGML_SCALE_MODE_COUNT, + _ => throw ArgumentError('Unknown value for ggml_scale_mode: $value'), + }; +} -const int __WATCHOS_8_3 = 80300; +enum ggml_sched_priority { + GGML_SCHED_PRIO_LOW(-1), + GGML_SCHED_PRIO_NORMAL(0), + GGML_SCHED_PRIO_MEDIUM(1), + GGML_SCHED_PRIO_HIGH(2), + GGML_SCHED_PRIO_REALTIME(3); -const int __WATCHOS_8_4 = 80400; + final int value; + const ggml_sched_priority(this.value); -const int __WATCHOS_8_5 = 80500; + static ggml_sched_priority fromValue(int value) => switch (value) { + -1 => GGML_SCHED_PRIO_LOW, + 0 => GGML_SCHED_PRIO_NORMAL, + 1 => GGML_SCHED_PRIO_MEDIUM, + 2 => GGML_SCHED_PRIO_HIGH, + 3 => GGML_SCHED_PRIO_REALTIME, + _ => + throw ArgumentError('Unknown value for ggml_sched_priority: $value'), + }; +} -const int __WATCHOS_8_6 = 80600; +enum ggml_sort_order { + GGML_SORT_ORDER_ASC(0), + GGML_SORT_ORDER_DESC(1); -const int __WATCHOS_8_7 = 80700; + final int value; + const ggml_sort_order(this.value); -const int __WATCHOS_8_8 = 80800; + static ggml_sort_order fromValue(int value) => switch (value) { + 0 => GGML_SORT_ORDER_ASC, + 1 => GGML_SORT_ORDER_DESC, + _ => throw ArgumentError('Unknown value for ggml_sort_order: $value'), + }; +} -const int __WATCHOS_9_0 = 90000; +enum ggml_status { + GGML_STATUS_ALLOC_FAILED(-2), + GGML_STATUS_FAILED(-1), + GGML_STATUS_SUCCESS(0), + GGML_STATUS_ABORTED(1); -const int __WATCHOS_9_1 = 90100; + final int value; + const ggml_status(this.value); -const int __WATCHOS_9_2 = 90200; + static ggml_status fromValue(int value) => switch (value) { + -2 => GGML_STATUS_ALLOC_FAILED, + -1 => GGML_STATUS_FAILED, + 0 => GGML_STATUS_SUCCESS, + 1 => GGML_STATUS_ABORTED, + _ => throw ArgumentError('Unknown value for ggml_status: $value'), + }; +} -const int __WATCHOS_9_3 = 90300; +final class ggml_tallocr extends ffi.Struct { + external ggml_backend_buffer_t buffer; -const int __WATCHOS_9_4 = 90400; + external ffi.Pointer base; -const int __WATCHOS_9_5 = 90500; + @ffi.Size() + external int alignment; -const int __WATCHOS_9_6 = 90600; + @ffi.Size() + external int offset; -const int __WATCHOS_10_0 = 100000; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ggml_backend_buffer_t buffer, + required ffi.Pointer base, + required int alignment, + required int offset, + }) => + $allocator() + ..ref.buffer = buffer + ..ref.base = base + ..ref.alignment = alignment + ..ref.offset = offset; +} -const int __WATCHOS_10_1 = 100100; +final class ggml_tensor extends ffi.Struct { + @ffi.UnsignedInt() + external int typeAsInt; -const int __WATCHOS_10_2 = 100200; + ggml_type get type => ggml_type.fromValue(typeAsInt); + set type(ggml_type value) => typeAsInt = value.value; -const int __WATCHOS_10_3 = 100300; + external ffi.Pointer buffer; -const int __WATCHOS_10_4 = 100400; + @ffi.Array.multi([4]) + external ffi.Array ne; -const int __WATCHOS_10_5 = 100500; + @ffi.Array.multi([4]) + external ffi.Array nb; -const int __WATCHOS_10_6 = 100600; + @ffi.UnsignedInt() + external int opAsInt; -const int __WATCHOS_10_7 = 100700; + ggml_op get op => ggml_op.fromValue(opAsInt); + set op(ggml_op value) => opAsInt = value.value; -const int __WATCHOS_11_0 = 110000; + @ffi.Array.multi([16]) + external ffi.Array op_params; -const int __WATCHOS_11_1 = 110100; + @ffi.Int32() + external int flags; -const int __WATCHOS_11_2 = 110200; + @ffi.Array.multi([10]) + external ffi.Array> src; -const int __WATCHOS_11_3 = 110300; + external ffi.Pointer view_src; -const int __WATCHOS_11_4 = 110400; + @ffi.Size() + external int view_offs; -const int __WATCHOS_11_5 = 110500; + external ffi.Pointer data; -const int __WATCHOS_11_6 = 110600; + @ffi.Array.multi([64]) + external ffi.Array name; -const int __WATCHOS_12_0 = 120000; + external ffi.Pointer extra; -const int __WATCHOS_26_0 = 260000; + @ffi.Array.multi([8]) + external ffi.Array padding; +} -const int __WATCHOS_26_1 = 260100; +enum ggml_tensor_flag { + GGML_TENSOR_FLAG_INPUT(1), + GGML_TENSOR_FLAG_OUTPUT(2), + GGML_TENSOR_FLAG_PARAM(4), + GGML_TENSOR_FLAG_LOSS(8), + GGML_TENSOR_FLAG_COMPUTE(16); -const int __WATCHOS_26_2 = 260200; + final int value; + const ggml_tensor_flag(this.value); -const int __WATCHOS_26_3 = 260300; + static ggml_tensor_flag fromValue(int value) => switch (value) { + 1 => GGML_TENSOR_FLAG_INPUT, + 2 => GGML_TENSOR_FLAG_OUTPUT, + 4 => GGML_TENSOR_FLAG_PARAM, + 8 => GGML_TENSOR_FLAG_LOSS, + 16 => GGML_TENSOR_FLAG_COMPUTE, + _ => throw ArgumentError('Unknown value for ggml_tensor_flag: $value'), + }; +} -const int __WATCHOS_26_4 = 260400; +final class ggml_threadpool extends ffi.Opaque {} -const int __WATCHOS_26_5 = 260500; +final class ggml_threadpool_params extends ffi.Struct { + @ffi.Array.multi([512]) + external ffi.Array cpumask; -const int __TVOS_9_0 = 90000; + @ffi.Int() + external int n_threads; -const int __TVOS_9_1 = 90100; + @ffi.Int() + external int prioAsInt; -const int __TVOS_9_2 = 90200; + ggml_sched_priority get prio => ggml_sched_priority.fromValue(prioAsInt); + set prio(ggml_sched_priority value) => prioAsInt = value.value; -const int __TVOS_10_0 = 100000; + @ffi.Uint32() + external int poll; -const int __TVOS_10_0_1 = 100001; + @ffi.Bool() + external bool strict_cpu; -const int __TVOS_10_1 = 100100; + @ffi.Bool() + external bool paused; +} -const int __TVOS_10_2 = 100200; +typedef ggml_threadpool_t = ffi.Pointer; +typedef ggml_to_float_t + = ffi.Pointer>; +typedef ggml_to_float_tFunction = ffi.Void Function( + ffi.Pointer x, ffi.Pointer y, ffi.Int64 k); +typedef Dartggml_to_float_tFunction = void Function( + ffi.Pointer x, ffi.Pointer y, int k); -const int __TVOS_11_0 = 110000; +enum ggml_tri_type { + GGML_TRI_TYPE_UPPER_DIAG(0), + GGML_TRI_TYPE_UPPER(1), + GGML_TRI_TYPE_LOWER_DIAG(2), + GGML_TRI_TYPE_LOWER(3); -const int __TVOS_11_1 = 110100; + final int value; + const ggml_tri_type(this.value); -const int __TVOS_11_2 = 110200; + static ggml_tri_type fromValue(int value) => switch (value) { + 0 => GGML_TRI_TYPE_UPPER_DIAG, + 1 => GGML_TRI_TYPE_UPPER, + 2 => GGML_TRI_TYPE_LOWER_DIAG, + 3 => GGML_TRI_TYPE_LOWER, + _ => throw ArgumentError('Unknown value for ggml_tri_type: $value'), + }; +} -const int __TVOS_11_3 = 110300; +enum ggml_type { + GGML_TYPE_F32(0), + GGML_TYPE_F16(1), + GGML_TYPE_Q4_0(2), + GGML_TYPE_Q4_1(3), + GGML_TYPE_Q5_0(6), + GGML_TYPE_Q5_1(7), + GGML_TYPE_Q8_0(8), + GGML_TYPE_Q8_1(9), + GGML_TYPE_Q2_K(10), + GGML_TYPE_Q3_K(11), + GGML_TYPE_Q4_K(12), + GGML_TYPE_Q5_K(13), + GGML_TYPE_Q6_K(14), + GGML_TYPE_Q8_K(15), + GGML_TYPE_IQ2_XXS(16), + GGML_TYPE_IQ2_XS(17), + GGML_TYPE_IQ3_XXS(18), + GGML_TYPE_IQ1_S(19), + GGML_TYPE_IQ4_NL(20), + GGML_TYPE_IQ3_S(21), + GGML_TYPE_IQ2_S(22), + GGML_TYPE_IQ4_XS(23), + GGML_TYPE_I8(24), + GGML_TYPE_I16(25), + GGML_TYPE_I32(26), + GGML_TYPE_I64(27), + GGML_TYPE_F64(28), + GGML_TYPE_IQ1_M(29), + GGML_TYPE_BF16(30), + GGML_TYPE_TQ1_0(34), + GGML_TYPE_TQ2_0(35), + GGML_TYPE_MXFP4(39), + GGML_TYPE_NVFP4(40), + GGML_TYPE_Q1_0(41), + GGML_TYPE_Q2_0(42), + GGML_TYPE_COUNT(43); -const int __TVOS_11_4 = 110400; + final int value; + const ggml_type(this.value); -const int __TVOS_12_0 = 120000; + static ggml_type fromValue(int value) => switch (value) { + 0 => GGML_TYPE_F32, + 1 => GGML_TYPE_F16, + 2 => GGML_TYPE_Q4_0, + 3 => GGML_TYPE_Q4_1, + 6 => GGML_TYPE_Q5_0, + 7 => GGML_TYPE_Q5_1, + 8 => GGML_TYPE_Q8_0, + 9 => GGML_TYPE_Q8_1, + 10 => GGML_TYPE_Q2_K, + 11 => GGML_TYPE_Q3_K, + 12 => GGML_TYPE_Q4_K, + 13 => GGML_TYPE_Q5_K, + 14 => GGML_TYPE_Q6_K, + 15 => GGML_TYPE_Q8_K, + 16 => GGML_TYPE_IQ2_XXS, + 17 => GGML_TYPE_IQ2_XS, + 18 => GGML_TYPE_IQ3_XXS, + 19 => GGML_TYPE_IQ1_S, + 20 => GGML_TYPE_IQ4_NL, + 21 => GGML_TYPE_IQ3_S, + 22 => GGML_TYPE_IQ2_S, + 23 => GGML_TYPE_IQ4_XS, + 24 => GGML_TYPE_I8, + 25 => GGML_TYPE_I16, + 26 => GGML_TYPE_I32, + 27 => GGML_TYPE_I64, + 28 => GGML_TYPE_F64, + 29 => GGML_TYPE_IQ1_M, + 30 => GGML_TYPE_BF16, + 34 => GGML_TYPE_TQ1_0, + 35 => GGML_TYPE_TQ2_0, + 39 => GGML_TYPE_MXFP4, + 40 => GGML_TYPE_NVFP4, + 41 => GGML_TYPE_Q1_0, + 42 => GGML_TYPE_Q2_0, + 43 => GGML_TYPE_COUNT, + _ => throw ArgumentError('Unknown value for ggml_type: $value'), + }; +} -const int __TVOS_12_1 = 120100; +final class ggml_type_traits extends ffi.Struct { + external ffi.Pointer type_name; -const int __TVOS_12_2 = 120200; + @ffi.Int64() + external int blck_size; -const int __TVOS_12_3 = 120300; + @ffi.Int64() + external int blck_size_interleave; -const int __TVOS_12_4 = 120400; + @ffi.Size() + external int type_size; -const int __TVOS_13_0 = 130000; + @ffi.Bool() + external bool is_quantized; -const int __TVOS_13_2 = 130200; + external ggml_to_float_t to_float; -const int __TVOS_13_3 = 130300; + external ggml_from_float_t from_float_ref; -const int __TVOS_13_4 = 130400; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer type_name, + required int blck_size, + required int blck_size_interleave, + required int type_size, + required bool is_quantized, + required ggml_to_float_t to_float, + required ggml_from_float_t from_float_ref, + }) => + $allocator() + ..ref.type_name = type_name + ..ref.blck_size = blck_size + ..ref.blck_size_interleave = blck_size_interleave + ..ref.type_size = type_size + ..ref.is_quantized = is_quantized + ..ref.to_float = to_float + ..ref.from_float_ref = from_float_ref; +} -const int __TVOS_14_0 = 140000; +final class ggml_type_traits_cpu extends ffi.Struct { + external ggml_from_float_t from_float; -const int __TVOS_14_1 = 140100; + external ggml_vec_dot_t vec_dot; -const int __TVOS_14_2 = 140200; + @ffi.UnsignedInt() + external int vec_dot_typeAsInt; -const int __TVOS_14_3 = 140300; + ggml_type get vec_dot_type => ggml_type.fromValue(vec_dot_typeAsInt); + set vec_dot_type(ggml_type value) => vec_dot_typeAsInt = value.value; -const int __TVOS_14_5 = 140500; + @ffi.Int64() + external int nrows; -const int __TVOS_14_6 = 140600; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ggml_from_float_t from_float, + required ggml_vec_dot_t vec_dot, + required ggml_type vec_dot_type, + required int nrows, + }) => + $allocator() + ..ref.from_float = from_float + ..ref.vec_dot = vec_dot + ..ref.vec_dot_type = vec_dot_type + ..ref.nrows = nrows; +} -const int __TVOS_14_7 = 140700; +enum ggml_unary_op { + GGML_UNARY_OP_ABS(0), + GGML_UNARY_OP_SGN(1), + GGML_UNARY_OP_NEG(2), + GGML_UNARY_OP_STEP(3), + GGML_UNARY_OP_TANH(4), + GGML_UNARY_OP_ELU(5), + GGML_UNARY_OP_RELU(6), + GGML_UNARY_OP_SIGMOID(7), + GGML_UNARY_OP_GELU(8), + GGML_UNARY_OP_GELU_QUICK(9), + GGML_UNARY_OP_SILU(10), + GGML_UNARY_OP_HARDSWISH(11), + GGML_UNARY_OP_HARDSIGMOID(12), + GGML_UNARY_OP_EXP(13), + GGML_UNARY_OP_EXPM1(14), + GGML_UNARY_OP_SOFTPLUS(15), + GGML_UNARY_OP_GELU_ERF(16), + GGML_UNARY_OP_XIELU(17), + GGML_UNARY_OP_FLOOR(18), + GGML_UNARY_OP_CEIL(19), + GGML_UNARY_OP_ROUND(20), + GGML_UNARY_OP_TRUNC(21), + GGML_UNARY_OP_COUNT(22); -const int __TVOS_15_0 = 150000; + final int value; + const ggml_unary_op(this.value); -const int __TVOS_15_1 = 150100; + static ggml_unary_op fromValue(int value) => switch (value) { + 0 => GGML_UNARY_OP_ABS, + 1 => GGML_UNARY_OP_SGN, + 2 => GGML_UNARY_OP_NEG, + 3 => GGML_UNARY_OP_STEP, + 4 => GGML_UNARY_OP_TANH, + 5 => GGML_UNARY_OP_ELU, + 6 => GGML_UNARY_OP_RELU, + 7 => GGML_UNARY_OP_SIGMOID, + 8 => GGML_UNARY_OP_GELU, + 9 => GGML_UNARY_OP_GELU_QUICK, + 10 => GGML_UNARY_OP_SILU, + 11 => GGML_UNARY_OP_HARDSWISH, + 12 => GGML_UNARY_OP_HARDSIGMOID, + 13 => GGML_UNARY_OP_EXP, + 14 => GGML_UNARY_OP_EXPM1, + 15 => GGML_UNARY_OP_SOFTPLUS, + 16 => GGML_UNARY_OP_GELU_ERF, + 17 => GGML_UNARY_OP_XIELU, + 18 => GGML_UNARY_OP_FLOOR, + 19 => GGML_UNARY_OP_CEIL, + 20 => GGML_UNARY_OP_ROUND, + 21 => GGML_UNARY_OP_TRUNC, + 22 => GGML_UNARY_OP_COUNT, + _ => throw ArgumentError('Unknown value for ggml_unary_op: $value'), + }; +} -const int __TVOS_15_2 = 150200; +typedef ggml_vec_dot_t + = ffi.Pointer>; +typedef ggml_vec_dot_tFunction = ffi.Void Function( + ffi.Int n, + ffi.Pointer s, + ffi.Size bs, + ffi.Pointer x, + ffi.Size bx, + ffi.Pointer y, + ffi.Size by, + ffi.Int nrc); +typedef Dartggml_vec_dot_tFunction = void Function( + int n, + ffi.Pointer s, + int bs, + ffi.Pointer x, + int bx, + ffi.Pointer y, + int by, + int nrc); -const int __TVOS_15_3 = 150300; +final class gguf_context extends ffi.Opaque {} -const int __TVOS_15_4 = 150400; +final class gguf_init_params extends ffi.Struct { + @ffi.Bool() + external bool no_alloc; -const int __TVOS_15_5 = 150500; + external ffi.Pointer> ctx; -const int __TVOS_15_6 = 150600; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required bool no_alloc, + required ffi.Pointer> ctx, + }) => + $allocator() + ..ref.no_alloc = no_alloc + ..ref.ctx = ctx; +} -const int __TVOS_16_0 = 160000; +typedef gguf_reader_callback_t + = ffi.Pointer>; +typedef gguf_reader_callback_tFunction = ffi.Size Function( + ffi.Pointer userdata, + ffi.Pointer output, + ffi.Uint64 offset, + ffi.Size len); +typedef Dartgguf_reader_callback_tFunction = int Function( + ffi.Pointer userdata, + ffi.Pointer output, + int offset, + int len); -const int __TVOS_16_1 = 160100; +enum gguf_type { + GGUF_TYPE_UINT8(0), + GGUF_TYPE_INT8(1), + GGUF_TYPE_UINT16(2), + GGUF_TYPE_INT16(3), + GGUF_TYPE_UINT32(4), + GGUF_TYPE_INT32(5), + GGUF_TYPE_FLOAT32(6), + GGUF_TYPE_BOOL(7), + GGUF_TYPE_STRING(8), + GGUF_TYPE_ARRAY(9), + GGUF_TYPE_UINT64(10), + GGUF_TYPE_INT64(11), + GGUF_TYPE_FLOAT64(12), + GGUF_TYPE_COUNT(13); -const int __TVOS_16_2 = 160200; + final int value; + const gguf_type(this.value); -const int __TVOS_16_3 = 160300; + static gguf_type fromValue(int value) => switch (value) { + 0 => GGUF_TYPE_UINT8, + 1 => GGUF_TYPE_INT8, + 2 => GGUF_TYPE_UINT16, + 3 => GGUF_TYPE_INT16, + 4 => GGUF_TYPE_UINT32, + 5 => GGUF_TYPE_INT32, + 6 => GGUF_TYPE_FLOAT32, + 7 => GGUF_TYPE_BOOL, + 8 => GGUF_TYPE_STRING, + 9 => GGUF_TYPE_ARRAY, + 10 => GGUF_TYPE_UINT64, + 11 => GGUF_TYPE_INT64, + 12 => GGUF_TYPE_FLOAT64, + 13 => GGUF_TYPE_COUNT, + _ => throw ArgumentError('Unknown value for gguf_type: $value'), + }; +} -const int __TVOS_16_4 = 160400; +typedef int_fast16_t = ffi.Int16; +typedef Dartint_fast16_t = int; +typedef int_fast32_t = ffi.Int32; +typedef Dartint_fast32_t = int; +typedef int_fast64_t = ffi.Int64; +typedef Dartint_fast64_t = int; +typedef int_fast8_t = ffi.Int8; +typedef Dartint_fast8_t = int; +typedef int_least16_t = ffi.Int16; +typedef Dartint_least16_t = int; +typedef int_least32_t = ffi.Int32; +typedef Dartint_least32_t = int; +typedef int_least64_t = ffi.Int64; +typedef Dartint_least64_t = int; +typedef int_least8_t = ffi.Int8; +typedef Dartint_least8_t = int; +typedef intmax_t = ffi.Long; +typedef Dartintmax_t = int; -const int __TVOS_16_5 = 160500; +final class llama_adapter_lora extends ffi.Opaque {} -const int __TVOS_16_6 = 160600; +enum llama_attention_type { + LLAMA_ATTENTION_TYPE_UNSPECIFIED(-1), + LLAMA_ATTENTION_TYPE_CAUSAL(0), + LLAMA_ATTENTION_TYPE_NON_CAUSAL(1); -const int __TVOS_17_0 = 170000; + final int value; + const llama_attention_type(this.value); -const int __TVOS_17_1 = 170100; + static llama_attention_type fromValue(int value) => switch (value) { + -1 => LLAMA_ATTENTION_TYPE_UNSPECIFIED, + 0 => LLAMA_ATTENTION_TYPE_CAUSAL, + 1 => LLAMA_ATTENTION_TYPE_NON_CAUSAL, + _ => + throw ArgumentError('Unknown value for llama_attention_type: $value'), + }; +} -const int __TVOS_17_2 = 170200; +final class llama_batch extends ffi.Struct { + @ffi.Int32() + external int n_tokens; -const int __TVOS_17_3 = 170300; + external ffi.Pointer token; -const int __TVOS_17_4 = 170400; + external ffi.Pointer embd; -const int __TVOS_17_5 = 170500; + external ffi.Pointer pos; -const int __TVOS_17_6 = 170600; + external ffi.Pointer n_seq_id; -const int __TVOS_18_0 = 180000; + external ffi.Pointer> seq_id; -const int __TVOS_18_1 = 180100; + external ffi.Pointer logits; -const int __TVOS_18_2 = 180200; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int n_tokens, + required ffi.Pointer token, + required ffi.Pointer embd, + required ffi.Pointer pos, + required ffi.Pointer n_seq_id, + required ffi.Pointer> seq_id, + required ffi.Pointer logits, + }) => + $allocator() + ..ref.n_tokens = n_tokens + ..ref.token = token + ..ref.embd = embd + ..ref.pos = pos + ..ref.n_seq_id = n_seq_id + ..ref.seq_id = seq_id + ..ref.logits = logits; +} -const int __TVOS_18_3 = 180300; +final class llama_chat_message extends ffi.Struct { + external ffi.Pointer role; -const int __TVOS_18_4 = 180400; + external ffi.Pointer content; -const int __TVOS_18_5 = 180500; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer role, + required ffi.Pointer content, + }) => + $allocator() + ..ref.role = role + ..ref.content = content; +} -const int __TVOS_18_6 = 180600; +final class llama_context extends ffi.Opaque {} -const int __TVOS_19_0 = 190000; +final class llama_context_params extends ffi.Struct { + @ffi.Uint32() + external int n_ctx; -const int __TVOS_26_0 = 260000; + @ffi.Uint32() + external int n_batch; -const int __TVOS_26_1 = 260100; + @ffi.Uint32() + external int n_ubatch; -const int __TVOS_26_2 = 260200; + @ffi.Uint32() + external int n_seq_max; -const int __TVOS_26_3 = 260300; + @ffi.Uint32() + external int n_rs_seq; -const int __TVOS_26_4 = 260400; + @ffi.Uint32() + external int n_outputs_max; -const int __TVOS_26_5 = 260500; + @ffi.Int32() + external int n_threads; -const int __BRIDGEOS_2_0 = 20000; + @ffi.Int32() + external int n_threads_batch; -const int __BRIDGEOS_3_0 = 30000; + @ffi.UnsignedInt() + external int ctx_typeAsInt; -const int __BRIDGEOS_3_1 = 30100; + llama_context_type get ctx_type => + llama_context_type.fromValue(ctx_typeAsInt); + set ctx_type(llama_context_type value) => ctx_typeAsInt = value.value; -const int __BRIDGEOS_3_4 = 30400; + @ffi.Int() + external int rope_scaling_typeAsInt; -const int __BRIDGEOS_4_0 = 40000; + llama_rope_scaling_type get rope_scaling_type => + llama_rope_scaling_type.fromValue(rope_scaling_typeAsInt); + set rope_scaling_type(llama_rope_scaling_type value) => + rope_scaling_typeAsInt = value.value; -const int __BRIDGEOS_4_1 = 40100; + @ffi.Int() + external int pooling_typeAsInt; -const int __BRIDGEOS_5_0 = 50000; + llama_pooling_type get pooling_type => + llama_pooling_type.fromValue(pooling_typeAsInt); + set pooling_type(llama_pooling_type value) => pooling_typeAsInt = value.value; -const int __BRIDGEOS_5_1 = 50100; + @ffi.Int() + external int attention_typeAsInt; -const int __BRIDGEOS_5_3 = 50300; + llama_attention_type get attention_type => + llama_attention_type.fromValue(attention_typeAsInt); + set attention_type(llama_attention_type value) => + attention_typeAsInt = value.value; -const int __BRIDGEOS_6_0 = 60000; + @ffi.Int() + external int flash_attn_typeAsInt; -const int __BRIDGEOS_6_2 = 60200; + llama_flash_attn_type get flash_attn_type => + llama_flash_attn_type.fromValue(flash_attn_typeAsInt); + set flash_attn_type(llama_flash_attn_type value) => + flash_attn_typeAsInt = value.value; -const int __BRIDGEOS_6_4 = 60400; + @ffi.Float() + external double rope_freq_base; -const int __BRIDGEOS_6_5 = 60500; + @ffi.Float() + external double rope_freq_scale; -const int __BRIDGEOS_6_6 = 60600; + @ffi.Float() + external double yarn_ext_factor; -const int __BRIDGEOS_7_0 = 70000; + @ffi.Float() + external double yarn_attn_factor; -const int __BRIDGEOS_7_1 = 70100; + @ffi.Float() + external double yarn_beta_fast; -const int __BRIDGEOS_7_2 = 70200; + @ffi.Float() + external double yarn_beta_slow; -const int __BRIDGEOS_7_3 = 70300; + @ffi.Uint32() + external int yarn_orig_ctx; -const int __BRIDGEOS_7_4 = 70400; + @ffi.Float() + external double defrag_thold; -const int __BRIDGEOS_7_6 = 70600; + external ggml_backend_sched_eval_callback cb_eval; -const int __BRIDGEOS_8_0 = 80000; + external ffi.Pointer cb_eval_user_data; -const int __BRIDGEOS_8_1 = 80100; + @ffi.UnsignedInt() + external int type_kAsInt; -const int __BRIDGEOS_8_2 = 80200; + ggml_type get type_k => ggml_type.fromValue(type_kAsInt); + set type_k(ggml_type value) => type_kAsInt = value.value; -const int __BRIDGEOS_8_3 = 80300; + @ffi.UnsignedInt() + external int type_vAsInt; -const int __BRIDGEOS_8_4 = 80400; + ggml_type get type_v => ggml_type.fromValue(type_vAsInt); + set type_v(ggml_type value) => type_vAsInt = value.value; -const int __BRIDGEOS_8_5 = 80500; + external ggml_abort_callback abort_callback; -const int __BRIDGEOS_8_6 = 80600; + external ffi.Pointer abort_callback_data; -const int __BRIDGEOS_9_0 = 90000; + @ffi.Bool() + external bool embeddings; -const int __BRIDGEOS_9_1 = 90100; + @ffi.Bool() + external bool offload_kqv; -const int __BRIDGEOS_9_2 = 90200; + @ffi.Bool() + external bool no_perf; -const int __BRIDGEOS_9_3 = 90300; + @ffi.Bool() + external bool op_offload; -const int __BRIDGEOS_9_4 = 90400; + @ffi.Bool() + external bool swa_full; -const int __BRIDGEOS_9_5 = 90500; + @ffi.Bool() + external bool kv_unified; -const int __BRIDGEOS_9_6 = 90600; + external ffi.Pointer samplers; -const int __BRIDGEOS_10_0 = 100000; + @ffi.Size() + external int n_samplers; -const int __BRIDGEOS_10_1 = 100100; + external ffi.Pointer ctx_other; -const int __BRIDGEOS_10_2 = 100200; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int n_ctx, + required int n_batch, + required int n_ubatch, + required int n_seq_max, + required int n_rs_seq, + required int n_outputs_max, + required int n_threads, + required int n_threads_batch, + required llama_context_type ctx_type, + required llama_rope_scaling_type rope_scaling_type, + required llama_pooling_type pooling_type, + required llama_attention_type attention_type, + required llama_flash_attn_type flash_attn_type, + required double rope_freq_base, + required double rope_freq_scale, + required double yarn_ext_factor, + required double yarn_attn_factor, + required double yarn_beta_fast, + required double yarn_beta_slow, + required int yarn_orig_ctx, + required double defrag_thold, + required ggml_backend_sched_eval_callback cb_eval, + required ffi.Pointer cb_eval_user_data, + required ggml_type type_k, + required ggml_type type_v, + required ggml_abort_callback abort_callback, + required ffi.Pointer abort_callback_data, + required bool embeddings, + required bool offload_kqv, + required bool no_perf, + required bool op_offload, + required bool swa_full, + required bool kv_unified, + required ffi.Pointer samplers, + required int n_samplers, + required ffi.Pointer ctx_other, + }) => + $allocator() + ..ref.n_ctx = n_ctx + ..ref.n_batch = n_batch + ..ref.n_ubatch = n_ubatch + ..ref.n_seq_max = n_seq_max + ..ref.n_rs_seq = n_rs_seq + ..ref.n_outputs_max = n_outputs_max + ..ref.n_threads = n_threads + ..ref.n_threads_batch = n_threads_batch + ..ref.ctx_type = ctx_type + ..ref.rope_scaling_type = rope_scaling_type + ..ref.pooling_type = pooling_type + ..ref.attention_type = attention_type + ..ref.flash_attn_type = flash_attn_type + ..ref.rope_freq_base = rope_freq_base + ..ref.rope_freq_scale = rope_freq_scale + ..ref.yarn_ext_factor = yarn_ext_factor + ..ref.yarn_attn_factor = yarn_attn_factor + ..ref.yarn_beta_fast = yarn_beta_fast + ..ref.yarn_beta_slow = yarn_beta_slow + ..ref.yarn_orig_ctx = yarn_orig_ctx + ..ref.defrag_thold = defrag_thold + ..ref.cb_eval = cb_eval + ..ref.cb_eval_user_data = cb_eval_user_data + ..ref.type_k = type_k + ..ref.type_v = type_v + ..ref.abort_callback = abort_callback + ..ref.abort_callback_data = abort_callback_data + ..ref.embeddings = embeddings + ..ref.offload_kqv = offload_kqv + ..ref.no_perf = no_perf + ..ref.op_offload = op_offload + ..ref.swa_full = swa_full + ..ref.kv_unified = kv_unified + ..ref.samplers = samplers + ..ref.n_samplers = n_samplers + ..ref.ctx_other = ctx_other; +} -const int __BRIDGEOS_10_3 = 100300; +enum llama_context_type { + LLAMA_CONTEXT_TYPE_DEFAULT(0), + LLAMA_CONTEXT_TYPE_MTP(1); -const int __BRIDGEOS_10_4 = 100400; + final int value; + const llama_context_type(this.value); -const int __BRIDGEOS_26_5 = 260500; + static llama_context_type fromValue(int value) => switch (value) { + 0 => LLAMA_CONTEXT_TYPE_DEFAULT, + 1 => LLAMA_CONTEXT_TYPE_MTP, + _ => + throw ArgumentError('Unknown value for llama_context_type: $value'), + }; +} -const int __DRIVERKIT_19_0 = 190000; +enum llama_flash_attn_type { + LLAMA_FLASH_ATTN_TYPE_AUTO(-1), + LLAMA_FLASH_ATTN_TYPE_DISABLED(0), + LLAMA_FLASH_ATTN_TYPE_ENABLED(1); -const int __DRIVERKIT_20_0 = 200000; + final int value; + const llama_flash_attn_type(this.value); -const int __DRIVERKIT_21_0 = 210000; + static llama_flash_attn_type fromValue(int value) => switch (value) { + -1 => LLAMA_FLASH_ATTN_TYPE_AUTO, + 0 => LLAMA_FLASH_ATTN_TYPE_DISABLED, + 1 => LLAMA_FLASH_ATTN_TYPE_ENABLED, + _ => throw ArgumentError( + 'Unknown value for llama_flash_attn_type: $value'), + }; +} -const int __DRIVERKIT_22_0 = 220000; +enum llama_ftype { + LLAMA_FTYPE_ALL_F32(0), + LLAMA_FTYPE_MOSTLY_F16(1), + LLAMA_FTYPE_MOSTLY_Q4_0(2), + LLAMA_FTYPE_MOSTLY_Q4_1(3), + LLAMA_FTYPE_MOSTLY_Q8_0(7), + LLAMA_FTYPE_MOSTLY_Q5_0(8), + LLAMA_FTYPE_MOSTLY_Q5_1(9), + LLAMA_FTYPE_MOSTLY_Q2_K(10), + LLAMA_FTYPE_MOSTLY_Q3_K_S(11), + LLAMA_FTYPE_MOSTLY_Q3_K_M(12), + LLAMA_FTYPE_MOSTLY_Q3_K_L(13), + LLAMA_FTYPE_MOSTLY_Q4_K_S(14), + LLAMA_FTYPE_MOSTLY_Q4_K_M(15), + LLAMA_FTYPE_MOSTLY_Q5_K_S(16), + LLAMA_FTYPE_MOSTLY_Q5_K_M(17), + LLAMA_FTYPE_MOSTLY_Q6_K(18), + LLAMA_FTYPE_MOSTLY_IQ2_XXS(19), + LLAMA_FTYPE_MOSTLY_IQ2_XS(20), + LLAMA_FTYPE_MOSTLY_Q2_K_S(21), + LLAMA_FTYPE_MOSTLY_IQ3_XS(22), + LLAMA_FTYPE_MOSTLY_IQ3_XXS(23), + LLAMA_FTYPE_MOSTLY_IQ1_S(24), + LLAMA_FTYPE_MOSTLY_IQ4_NL(25), + LLAMA_FTYPE_MOSTLY_IQ3_S(26), + LLAMA_FTYPE_MOSTLY_IQ3_M(27), + LLAMA_FTYPE_MOSTLY_IQ2_S(28), + LLAMA_FTYPE_MOSTLY_IQ2_M(29), + LLAMA_FTYPE_MOSTLY_IQ4_XS(30), + LLAMA_FTYPE_MOSTLY_IQ1_M(31), + LLAMA_FTYPE_MOSTLY_BF16(32), + LLAMA_FTYPE_MOSTLY_TQ1_0(36), + LLAMA_FTYPE_MOSTLY_TQ2_0(37), + LLAMA_FTYPE_MOSTLY_MXFP4_MOE(38), + LLAMA_FTYPE_MOSTLY_NVFP4(39), + LLAMA_FTYPE_MOSTLY_Q1_0(40), + LLAMA_FTYPE_MOSTLY_Q2_0(41), + LLAMA_FTYPE_GUESSED(1024); -const int __DRIVERKIT_22_4 = 220400; + final int value; + const llama_ftype(this.value); -const int __DRIVERKIT_22_5 = 220500; + static llama_ftype fromValue(int value) => switch (value) { + 0 => LLAMA_FTYPE_ALL_F32, + 1 => LLAMA_FTYPE_MOSTLY_F16, + 2 => LLAMA_FTYPE_MOSTLY_Q4_0, + 3 => LLAMA_FTYPE_MOSTLY_Q4_1, + 7 => LLAMA_FTYPE_MOSTLY_Q8_0, + 8 => LLAMA_FTYPE_MOSTLY_Q5_0, + 9 => LLAMA_FTYPE_MOSTLY_Q5_1, + 10 => LLAMA_FTYPE_MOSTLY_Q2_K, + 11 => LLAMA_FTYPE_MOSTLY_Q3_K_S, + 12 => LLAMA_FTYPE_MOSTLY_Q3_K_M, + 13 => LLAMA_FTYPE_MOSTLY_Q3_K_L, + 14 => LLAMA_FTYPE_MOSTLY_Q4_K_S, + 15 => LLAMA_FTYPE_MOSTLY_Q4_K_M, + 16 => LLAMA_FTYPE_MOSTLY_Q5_K_S, + 17 => LLAMA_FTYPE_MOSTLY_Q5_K_M, + 18 => LLAMA_FTYPE_MOSTLY_Q6_K, + 19 => LLAMA_FTYPE_MOSTLY_IQ2_XXS, + 20 => LLAMA_FTYPE_MOSTLY_IQ2_XS, + 21 => LLAMA_FTYPE_MOSTLY_Q2_K_S, + 22 => LLAMA_FTYPE_MOSTLY_IQ3_XS, + 23 => LLAMA_FTYPE_MOSTLY_IQ3_XXS, + 24 => LLAMA_FTYPE_MOSTLY_IQ1_S, + 25 => LLAMA_FTYPE_MOSTLY_IQ4_NL, + 26 => LLAMA_FTYPE_MOSTLY_IQ3_S, + 27 => LLAMA_FTYPE_MOSTLY_IQ3_M, + 28 => LLAMA_FTYPE_MOSTLY_IQ2_S, + 29 => LLAMA_FTYPE_MOSTLY_IQ2_M, + 30 => LLAMA_FTYPE_MOSTLY_IQ4_XS, + 31 => LLAMA_FTYPE_MOSTLY_IQ1_M, + 32 => LLAMA_FTYPE_MOSTLY_BF16, + 36 => LLAMA_FTYPE_MOSTLY_TQ1_0, + 37 => LLAMA_FTYPE_MOSTLY_TQ2_0, + 38 => LLAMA_FTYPE_MOSTLY_MXFP4_MOE, + 39 => LLAMA_FTYPE_MOSTLY_NVFP4, + 40 => LLAMA_FTYPE_MOSTLY_Q1_0, + 41 => LLAMA_FTYPE_MOSTLY_Q2_0, + 1024 => LLAMA_FTYPE_GUESSED, + _ => throw ArgumentError('Unknown value for llama_ftype: $value'), + }; +} -const int __DRIVERKIT_22_6 = 220600; +enum llama_load_mode { + LLAMA_LOAD_MODE_NONE(0), + LLAMA_LOAD_MODE_MMAP(1), + LLAMA_LOAD_MODE_MLOCK(2), + LLAMA_LOAD_MODE_MMAP_MLOCK(3), + LLAMA_LOAD_MODE_DIRECT_IO(4); -const int __DRIVERKIT_23_0 = 230000; + final int value; + const llama_load_mode(this.value); + + static llama_load_mode fromValue(int value) => switch (value) { + 0 => LLAMA_LOAD_MODE_NONE, + 1 => LLAMA_LOAD_MODE_MMAP, + 2 => LLAMA_LOAD_MODE_MLOCK, + 3 => LLAMA_LOAD_MODE_MMAP_MLOCK, + 4 => LLAMA_LOAD_MODE_DIRECT_IO, + _ => throw ArgumentError('Unknown value for llama_load_mode: $value'), + }; +} -const int __DRIVERKIT_23_1 = 230100; +final class llama_logit_bias extends ffi.Struct { + @llama_token() + external int token; -const int __DRIVERKIT_23_2 = 230200; + @ffi.Float() + external double bias; -const int __DRIVERKIT_23_3 = 230300; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int token, + required double bias, + }) => + $allocator() + ..ref.token = token + ..ref.bias = bias; +} -const int __DRIVERKIT_23_4 = 230400; +final class llama_memory_i extends ffi.Opaque {} -const int __DRIVERKIT_23_5 = 230500; +typedef llama_memory_t = ffi.Pointer; -const int __DRIVERKIT_23_6 = 230600; +final class llama_model extends ffi.Opaque {} -const int __DRIVERKIT_24_0 = 240000; +final class llama_model_imatrix_data extends ffi.Struct { + external ffi.Pointer name; -const int __DRIVERKIT_24_1 = 240100; + external ffi.Pointer data; -const int __DRIVERKIT_24_2 = 240200; + @ffi.Size() + external int size; -const int __DRIVERKIT_24_3 = 240300; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer name, + required ffi.Pointer data, + required int size, + }) => + $allocator() + ..ref.name = name + ..ref.data = data + ..ref.size = size; +} -const int __DRIVERKIT_24_4 = 240400; +final class llama_model_kv_override extends ffi.Struct { + @ffi.UnsignedInt() + external int tagAsInt; -const int __DRIVERKIT_24_5 = 240500; + llama_model_kv_override_type get tag => + llama_model_kv_override_type.fromValue(tagAsInt); + set tag(llama_model_kv_override_type value) => tagAsInt = value.value; -const int __DRIVERKIT_24_6 = 240600; + @ffi.Array.multi([128]) + external ffi.Array key; -const int __DRIVERKIT_25_0 = 250000; + external UnnamedUnion unnamed; +} -const int __DRIVERKIT_25_1 = 250100; +enum llama_model_kv_override_type { + LLAMA_KV_OVERRIDE_TYPE_INT(0), + LLAMA_KV_OVERRIDE_TYPE_FLOAT(1), + LLAMA_KV_OVERRIDE_TYPE_BOOL(2), + LLAMA_KV_OVERRIDE_TYPE_STR(3); -const int __DRIVERKIT_25_2 = 250200; + final int value; + const llama_model_kv_override_type(this.value); -const int __DRIVERKIT_25_3 = 250300; + static llama_model_kv_override_type fromValue(int value) => switch (value) { + 0 => LLAMA_KV_OVERRIDE_TYPE_INT, + 1 => LLAMA_KV_OVERRIDE_TYPE_FLOAT, + 2 => LLAMA_KV_OVERRIDE_TYPE_BOOL, + 3 => LLAMA_KV_OVERRIDE_TYPE_STR, + _ => throw ArgumentError( + 'Unknown value for llama_model_kv_override_type: $value'), + }; +} -const int __DRIVERKIT_25_4 = 250400; +enum llama_model_meta_key { + LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE(0), + LLAMA_MODEL_META_KEY_SAMPLING_TOP_K(1), + LLAMA_MODEL_META_KEY_SAMPLING_TOP_P(2), + LLAMA_MODEL_META_KEY_SAMPLING_MIN_P(3), + LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY(4), + LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD(5), + LLAMA_MODEL_META_KEY_SAMPLING_TEMP(6), + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N(7), + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT(8), + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT(9), + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU(10), + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA(11); -const int __DRIVERKIT_25_5 = 250500; + final int value; + const llama_model_meta_key(this.value); -const int __VISIONOS_1_0 = 10000; + static llama_model_meta_key fromValue(int value) => switch (value) { + 0 => LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE, + 1 => LLAMA_MODEL_META_KEY_SAMPLING_TOP_K, + 2 => LLAMA_MODEL_META_KEY_SAMPLING_TOP_P, + 3 => LLAMA_MODEL_META_KEY_SAMPLING_MIN_P, + 4 => LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY, + 5 => LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD, + 6 => LLAMA_MODEL_META_KEY_SAMPLING_TEMP, + 7 => LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N, + 8 => LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT, + 9 => LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT, + 10 => LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU, + 11 => LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA, + _ => + throw ArgumentError('Unknown value for llama_model_meta_key: $value'), + }; +} -const int __VISIONOS_1_1 = 10100; +final class llama_model_params extends ffi.Struct { + external ffi.Pointer devices; -const int __VISIONOS_1_2 = 10200; + external ffi.Pointer tensor_buft_overrides; -const int __VISIONOS_1_3 = 10300; + @ffi.Int32() + external int n_gpu_layers; -const int __VISIONOS_2_0 = 20000; + @ffi.UnsignedInt() + external int split_modeAsInt; -const int __VISIONOS_2_1 = 20100; + llama_split_mode get split_mode => + llama_split_mode.fromValue(split_modeAsInt); + set split_mode(llama_split_mode value) => split_modeAsInt = value.value; -const int __VISIONOS_2_2 = 20200; + @ffi.UnsignedInt() + external int load_modeAsInt; -const int __VISIONOS_2_3 = 20300; + llama_load_mode get load_mode => llama_load_mode.fromValue(load_modeAsInt); + set load_mode(llama_load_mode value) => load_modeAsInt = value.value; -const int __VISIONOS_2_4 = 20400; + @ffi.Int32() + external int main_gpu; -const int __VISIONOS_2_5 = 20500; + external ffi.Pointer tensor_split; -const int __VISIONOS_2_6 = 20600; + external llama_progress_callback progress_callback; -const int __VISIONOS_3_0 = 30000; + external ffi.Pointer progress_callback_user_data; -const int __VISIONOS_26_0 = 260000; + external ffi.Pointer kv_overrides; -const int __VISIONOS_26_1 = 260100; + @ffi.Bool() + external bool vocab_only; -const int __VISIONOS_26_2 = 260200; + @ffi.Bool() + external bool check_tensors; -const int __VISIONOS_26_3 = 260300; + @ffi.Bool() + external bool use_extra_bufts; -const int __VISIONOS_26_4 = 260400; + @ffi.Bool() + external bool no_host; -const int __VISIONOS_26_5 = 260500; + @ffi.Bool() + external bool no_alloc; -const int MAC_OS_X_VERSION_10_0 = 1000; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer devices, + required ffi.Pointer + tensor_buft_overrides, + required int n_gpu_layers, + required llama_split_mode split_mode, + required llama_load_mode load_mode, + required int main_gpu, + required ffi.Pointer tensor_split, + required llama_progress_callback progress_callback, + required ffi.Pointer progress_callback_user_data, + required ffi.Pointer kv_overrides, + required bool vocab_only, + required bool check_tensors, + required bool use_extra_bufts, + required bool no_host, + required bool no_alloc, + }) => + $allocator() + ..ref.devices = devices + ..ref.tensor_buft_overrides = tensor_buft_overrides + ..ref.n_gpu_layers = n_gpu_layers + ..ref.split_mode = split_mode + ..ref.load_mode = load_mode + ..ref.main_gpu = main_gpu + ..ref.tensor_split = tensor_split + ..ref.progress_callback = progress_callback + ..ref.progress_callback_user_data = progress_callback_user_data + ..ref.kv_overrides = kv_overrides + ..ref.vocab_only = vocab_only + ..ref.check_tensors = check_tensors + ..ref.use_extra_bufts = use_extra_bufts + ..ref.no_host = no_host + ..ref.no_alloc = no_alloc; +} -const int MAC_OS_X_VERSION_10_1 = 1010; +final class llama_model_quantize_params extends ffi.Struct { + @ffi.Int32() + external int nthread; -const int MAC_OS_X_VERSION_10_2 = 1020; + @ffi.UnsignedInt() + external int ftypeAsInt; -const int MAC_OS_X_VERSION_10_3 = 1030; + llama_ftype get ftype => llama_ftype.fromValue(ftypeAsInt); + set ftype(llama_ftype value) => ftypeAsInt = value.value; -const int MAC_OS_X_VERSION_10_4 = 1040; + @ffi.UnsignedInt() + external int output_tensor_typeAsInt; -const int MAC_OS_X_VERSION_10_5 = 1050; + ggml_type get output_tensor_type => + ggml_type.fromValue(output_tensor_typeAsInt); + set output_tensor_type(ggml_type value) => + output_tensor_typeAsInt = value.value; -const int MAC_OS_X_VERSION_10_6 = 1060; + @ffi.UnsignedInt() + external int token_embedding_typeAsInt; -const int MAC_OS_X_VERSION_10_7 = 1070; + ggml_type get token_embedding_type => + ggml_type.fromValue(token_embedding_typeAsInt); + set token_embedding_type(ggml_type value) => + token_embedding_typeAsInt = value.value; -const int MAC_OS_X_VERSION_10_8 = 1080; + @ffi.Bool() + external bool allow_requantize; -const int MAC_OS_X_VERSION_10_9 = 1090; + @ffi.Bool() + external bool quantize_output_tensor; -const int MAC_OS_X_VERSION_10_10 = 101000; + @ffi.Bool() + external bool only_copy; -const int MAC_OS_X_VERSION_10_10_2 = 101002; + @ffi.Bool() + external bool pure; -const int MAC_OS_X_VERSION_10_10_3 = 101003; + @ffi.Bool() + external bool keep_split; -const int MAC_OS_X_VERSION_10_11 = 101100; + @ffi.Bool() + external bool dry_run; -const int MAC_OS_X_VERSION_10_11_2 = 101102; + external ffi.Pointer imatrix; -const int MAC_OS_X_VERSION_10_11_3 = 101103; + external ffi.Pointer kv_overrides; -const int MAC_OS_X_VERSION_10_11_4 = 101104; + external ffi.Pointer tt_overrides; -const int MAC_OS_X_VERSION_10_12 = 101200; + external ffi.Pointer prune_layers; -const int MAC_OS_X_VERSION_10_12_1 = 101201; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int nthread, + required llama_ftype ftype, + required ggml_type output_tensor_type, + required ggml_type token_embedding_type, + required bool allow_requantize, + required bool quantize_output_tensor, + required bool only_copy, + required bool pure, + required bool keep_split, + required bool dry_run, + required ffi.Pointer imatrix, + required ffi.Pointer kv_overrides, + required ffi.Pointer tt_overrides, + required ffi.Pointer prune_layers, + }) => + $allocator() + ..ref.nthread = nthread + ..ref.ftype = ftype + ..ref.output_tensor_type = output_tensor_type + ..ref.token_embedding_type = token_embedding_type + ..ref.allow_requantize = allow_requantize + ..ref.quantize_output_tensor = quantize_output_tensor + ..ref.only_copy = only_copy + ..ref.pure = pure + ..ref.keep_split = keep_split + ..ref.dry_run = dry_run + ..ref.imatrix = imatrix + ..ref.kv_overrides = kv_overrides + ..ref.tt_overrides = tt_overrides + ..ref.prune_layers = prune_layers; +} -const int MAC_OS_X_VERSION_10_12_2 = 101202; +typedef llama_model_set_tensor_data_t + = ffi.Pointer>; +typedef llama_model_set_tensor_data_tFunction = ffi.Void Function( + ffi.Pointer tensor, ffi.Pointer userdata); +typedef Dartllama_model_set_tensor_data_tFunction = void Function( + ffi.Pointer tensor, ffi.Pointer userdata); -const int MAC_OS_X_VERSION_10_12_4 = 101204; +final class llama_model_tensor_buft_override extends ffi.Struct { + external ffi.Pointer pattern; -const int MAC_OS_X_VERSION_10_13 = 101300; + external ggml_backend_buffer_type_t buft; -const int MAC_OS_X_VERSION_10_13_1 = 101301; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer pattern, + required ggml_backend_buffer_type_t buft, + }) => + $allocator() + ..ref.pattern = pattern + ..ref.buft = buft; +} -const int MAC_OS_X_VERSION_10_13_2 = 101302; +final class llama_model_tensor_override extends ffi.Struct { + external ffi.Pointer pattern; -const int MAC_OS_X_VERSION_10_13_4 = 101304; + @ffi.UnsignedInt() + external int typeAsInt; -const int MAC_OS_X_VERSION_10_14 = 101400; + ggml_type get type => ggml_type.fromValue(typeAsInt); + set type(ggml_type value) => typeAsInt = value.value; + + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer pattern, + required ggml_type type, + }) => + $allocator() + ..ref.pattern = pattern + ..ref.type = type; +} -const int MAC_OS_X_VERSION_10_14_1 = 101401; +typedef llama_opt_param_filter + = ffi.Pointer>; +typedef llama_opt_param_filterFunction = ffi.Bool Function( + ffi.Pointer tensor, ffi.Pointer userdata); +typedef Dartllama_opt_param_filterFunction = bool Function( + ffi.Pointer tensor, ffi.Pointer userdata); -const int MAC_OS_X_VERSION_10_14_4 = 101404; +final class llama_opt_params extends ffi.Struct { + @ffi.Uint32() + external int n_ctx_train; -const int MAC_OS_X_VERSION_10_14_5 = 101405; + external llama_opt_param_filter param_filter; -const int MAC_OS_X_VERSION_10_14_6 = 101406; + external ffi.Pointer param_filter_ud; -const int MAC_OS_X_VERSION_10_15 = 101500; + external ggml_opt_get_optimizer_params get_opt_pars; -const int MAC_OS_X_VERSION_10_15_1 = 101501; + external ffi.Pointer get_opt_pars_ud; -const int MAC_OS_X_VERSION_10_15_4 = 101504; + @ffi.UnsignedInt() + external int optimizer_typeAsInt; -const int MAC_OS_X_VERSION_10_16 = 101600; + ggml_opt_optimizer_type get optimizer_type => + ggml_opt_optimizer_type.fromValue(optimizer_typeAsInt); + set optimizer_type(ggml_opt_optimizer_type value) => + optimizer_typeAsInt = value.value; + + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int n_ctx_train, + required llama_opt_param_filter param_filter, + required ffi.Pointer param_filter_ud, + required ggml_opt_get_optimizer_params get_opt_pars, + required ffi.Pointer get_opt_pars_ud, + required ggml_opt_optimizer_type optimizer_type, + }) => + $allocator() + ..ref.n_ctx_train = n_ctx_train + ..ref.param_filter = param_filter + ..ref.param_filter_ud = param_filter_ud + ..ref.get_opt_pars = get_opt_pars + ..ref.get_opt_pars_ud = get_opt_pars_ud + ..ref.optimizer_type = optimizer_type; +} -const int MAC_OS_VERSION_11_0 = 110000; +final class llama_perf_context_data extends ffi.Struct { + @ffi.Double() + external double t_start_ms; -const int MAC_OS_VERSION_11_1 = 110100; + @ffi.Double() + external double t_load_ms; -const int MAC_OS_VERSION_11_3 = 110300; + @ffi.Double() + external double t_p_eval_ms; -const int MAC_OS_VERSION_11_4 = 110400; + @ffi.Double() + external double t_eval_ms; -const int MAC_OS_VERSION_11_5 = 110500; + @ffi.Int32() + external int n_p_eval; -const int MAC_OS_VERSION_11_6 = 110600; + @ffi.Int32() + external int n_eval; -const int MAC_OS_VERSION_12_0 = 120000; + @ffi.Int32() + external int n_reused; -const int MAC_OS_VERSION_12_1 = 120100; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required double t_start_ms, + required double t_load_ms, + required double t_p_eval_ms, + required double t_eval_ms, + required int n_p_eval, + required int n_eval, + required int n_reused, + }) => + $allocator() + ..ref.t_start_ms = t_start_ms + ..ref.t_load_ms = t_load_ms + ..ref.t_p_eval_ms = t_p_eval_ms + ..ref.t_eval_ms = t_eval_ms + ..ref.n_p_eval = n_p_eval + ..ref.n_eval = n_eval + ..ref.n_reused = n_reused; +} -const int MAC_OS_VERSION_12_2 = 120200; +final class llama_perf_sampler_data extends ffi.Struct { + @ffi.Double() + external double t_sample_ms; -const int MAC_OS_VERSION_12_3 = 120300; + @ffi.Int32() + external int n_sample; -const int MAC_OS_VERSION_12_4 = 120400; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required double t_sample_ms, + required int n_sample, + }) => + $allocator() + ..ref.t_sample_ms = t_sample_ms + ..ref.n_sample = n_sample; +} -const int MAC_OS_VERSION_12_5 = 120500; +enum llama_pooling_type { + LLAMA_POOLING_TYPE_UNSPECIFIED(-1), + LLAMA_POOLING_TYPE_NONE(0), + LLAMA_POOLING_TYPE_MEAN(1), + LLAMA_POOLING_TYPE_CLS(2), + LLAMA_POOLING_TYPE_LAST(3), + LLAMA_POOLING_TYPE_RANK(4); -const int MAC_OS_VERSION_12_6 = 120600; + final int value; + const llama_pooling_type(this.value); -const int MAC_OS_VERSION_12_7 = 120700; + static llama_pooling_type fromValue(int value) => switch (value) { + -1 => LLAMA_POOLING_TYPE_UNSPECIFIED, + 0 => LLAMA_POOLING_TYPE_NONE, + 1 => LLAMA_POOLING_TYPE_MEAN, + 2 => LLAMA_POOLING_TYPE_CLS, + 3 => LLAMA_POOLING_TYPE_LAST, + 4 => LLAMA_POOLING_TYPE_RANK, + _ => + throw ArgumentError('Unknown value for llama_pooling_type: $value'), + }; +} -const int MAC_OS_VERSION_13_0 = 130000; +typedef llama_pos = ffi.Int32; +typedef Dartllama_pos = int; +typedef llama_progress_callback + = ffi.Pointer>; +typedef llama_progress_callbackFunction = ffi.Bool Function( + ffi.Float progress, ffi.Pointer user_data); +typedef Dartllama_progress_callbackFunction = bool Function( + double progress, ffi.Pointer user_data); -const int MAC_OS_VERSION_13_1 = 130100; +enum llama_rope_scaling_type { + LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED(-1), + LLAMA_ROPE_SCALING_TYPE_NONE(0), + LLAMA_ROPE_SCALING_TYPE_LINEAR(1), + LLAMA_ROPE_SCALING_TYPE_YARN(2), + LLAMA_ROPE_SCALING_TYPE_LONGROPE(3); -const int MAC_OS_VERSION_13_2 = 130200; + static const LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = + LLAMA_ROPE_SCALING_TYPE_LONGROPE; -const int MAC_OS_VERSION_13_3 = 130300; + final int value; + const llama_rope_scaling_type(this.value); -const int MAC_OS_VERSION_13_4 = 130400; + static llama_rope_scaling_type fromValue(int value) => switch (value) { + -1 => LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, + 0 => LLAMA_ROPE_SCALING_TYPE_NONE, + 1 => LLAMA_ROPE_SCALING_TYPE_LINEAR, + 2 => LLAMA_ROPE_SCALING_TYPE_YARN, + 3 => LLAMA_ROPE_SCALING_TYPE_LONGROPE, + _ => throw ArgumentError( + 'Unknown value for llama_rope_scaling_type: $value'), + }; -const int MAC_OS_VERSION_13_5 = 130500; + @override + String toString() { + if (this == LLAMA_ROPE_SCALING_TYPE_LONGROPE) + return "llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_LONGROPE, llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_MAX_VALUE"; + return super.toString(); + } +} -const int MAC_OS_VERSION_13_6 = 130600; +enum llama_rope_type { + LLAMA_ROPE_TYPE_NONE(-1), + LLAMA_ROPE_TYPE_NORM(0), + LLAMA_ROPE_TYPE_NEOX(2), + LLAMA_ROPE_TYPE_MROPE(8), + LLAMA_ROPE_TYPE_IMROPE(40), + LLAMA_ROPE_TYPE_VISION(24); -const int MAC_OS_VERSION_13_7 = 130700; + final int value; + const llama_rope_type(this.value); -const int MAC_OS_VERSION_14_0 = 140000; + static llama_rope_type fromValue(int value) => switch (value) { + -1 => LLAMA_ROPE_TYPE_NONE, + 0 => LLAMA_ROPE_TYPE_NORM, + 2 => LLAMA_ROPE_TYPE_NEOX, + 8 => LLAMA_ROPE_TYPE_MROPE, + 40 => LLAMA_ROPE_TYPE_IMROPE, + 24 => LLAMA_ROPE_TYPE_VISION, + _ => throw ArgumentError('Unknown value for llama_rope_type: $value'), + }; +} -const int MAC_OS_VERSION_14_1 = 140100; +final class llama_sampler extends ffi.Struct { + external ffi.Pointer iface; -const int MAC_OS_VERSION_14_2 = 140200; + external llama_sampler_context_t ctx; -const int MAC_OS_VERSION_14_3 = 140300; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer iface, + required llama_sampler_context_t ctx, + }) => + $allocator() + ..ref.iface = iface + ..ref.ctx = ctx; +} -const int MAC_OS_VERSION_14_4 = 140400; +final class llama_sampler_chain_params extends ffi.Struct { + @ffi.Bool() + external bool no_perf; -const int MAC_OS_VERSION_14_5 = 140500; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required bool no_perf, + }) => + $allocator()..ref.no_perf = no_perf; +} -const int MAC_OS_VERSION_14_6 = 140600; +typedef llama_sampler_context_t = ffi.Pointer; -const int MAC_OS_VERSION_14_7 = 140700; +final class llama_sampler_data extends ffi.Struct { + external ffi.Pointer logits; -const int MAC_OS_VERSION_15_0 = 150000; + external ffi.Pointer probs; -const int MAC_OS_VERSION_15_1 = 150100; + external ffi.Pointer sampled; -const int MAC_OS_VERSION_15_2 = 150200; + external ffi.Pointer candidates; -const int MAC_OS_VERSION_15_3 = 150300; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer logits, + required ffi.Pointer probs, + required ffi.Pointer sampled, + required ffi.Pointer candidates, + }) => + $allocator() + ..ref.logits = logits + ..ref.probs = probs + ..ref.sampled = sampled + ..ref.candidates = candidates; +} -const int MAC_OS_VERSION_15_4 = 150400; +final class llama_sampler_i extends ffi.Struct { + external ffi.Pointer< + ffi.NativeFunction< + ffi.Pointer Function(ffi.Pointer smpl)>> + name; -const int MAC_OS_VERSION_15_5 = 150500; + external ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer smpl, llama_token token)>> accept; -const int MAC_OS_VERSION_15_6 = 150600; + external ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer smpl, + ffi.Pointer cur_p)>> apply; -const int MAC_OS_VERSION_16_0 = 160000; + external ffi.Pointer< + ffi + .NativeFunction smpl)>> + reset; -const int MAC_OS_VERSION_26_0 = 260000; + external ffi.Pointer< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer smpl)>> clone; -const int MAC_OS_VERSION_26_1 = 260100; + external ffi.Pointer< + ffi + .NativeFunction smpl)>> free; -const int MAC_OS_VERSION_26_2 = 260200; + external ffi.Pointer< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer smpl, + ggml_backend_buffer_type_t buft)>> backend_init; -const int MAC_OS_VERSION_26_3 = 260300; + external ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer smpl, + ffi.Pointer ctx, + ffi.Pointer gf, + ffi.Pointer selected_token)>> backend_accept; -const int MAC_OS_VERSION_26_4 = 260400; + external ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer smpl, + ffi.Pointer ctx, + ffi.Pointer gf, + ffi.Pointer data)>> backend_apply; -const int MAC_OS_VERSION_26_5 = 260500; + external ffi.Pointer< + ffi + .NativeFunction smpl)>> + backend_set_input; -const int __AVAILABILITY_VERSIONS_VERSION_HASH = 93585900; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer smpl)>> + name, + required ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer smpl, llama_token token)>> + accept, + required ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer smpl, + ffi.Pointer cur_p)>> + apply, + required ffi.Pointer< + ffi + .NativeFunction smpl)>> + reset, + required ffi.Pointer< + ffi.NativeFunction< + ffi.Pointer Function( + ffi.Pointer smpl)>> + clone, + required ffi.Pointer< + ffi + .NativeFunction smpl)>> + free, + required ffi.Pointer< + ffi.NativeFunction< + ffi.Bool Function(ffi.Pointer smpl, + ggml_backend_buffer_type_t buft)>> + backend_init, + required ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer smpl, + ffi.Pointer ctx, + ffi.Pointer gf, + ffi.Pointer selected_token)>> + backend_accept, + required ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function( + ffi.Pointer smpl, + ffi.Pointer ctx, + ffi.Pointer gf, + ffi.Pointer data)>> + backend_apply, + required ffi.Pointer< + ffi + .NativeFunction smpl)>> + backend_set_input, + }) => + $allocator() + ..ref.name = name + ..ref.accept = accept + ..ref.apply = apply + ..ref.reset = reset + ..ref.clone = clone + ..ref.free = free + ..ref.backend_init = backend_init + ..ref.backend_accept = backend_accept + ..ref.backend_apply = backend_apply + ..ref.backend_set_input = backend_set_input; +} -const String __AVAILABILITY_VERSIONS_VERSION_STRING = 'Local'; +final class llama_sampler_seq_config extends ffi.Struct { + @llama_seq_id() + external int seq_id; -const String __AVAILABILITY_FILE = 'AvailabilityVersions.h'; + external ffi.Pointer sampler; -const int __MAC_OS_X_VERSION_MIN_REQUIRED = 260000; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int seq_id, + required ffi.Pointer sampler, + }) => + $allocator() + ..ref.seq_id = seq_id + ..ref.sampler = sampler; +} -const int __MAC_OS_X_VERSION_MAX_ALLOWED = 260500; +typedef llama_seq_id = ffi.Int32; +typedef Dartllama_seq_id = int; -const int __ENABLE_LEGACY_MAC_AVAILABILITY = 1; +enum llama_split_mode { + LLAMA_SPLIT_MODE_NONE(0), + LLAMA_SPLIT_MODE_LAYER(1), + LLAMA_SPLIT_MODE_ROW(2), + LLAMA_SPLIT_MODE_TENSOR(3); -const int RENAME_SECLUDE = 1; + final int value; + const llama_split_mode(this.value); -const int RENAME_SWAP = 2; + static llama_split_mode fromValue(int value) => switch (value) { + 0 => LLAMA_SPLIT_MODE_NONE, + 1 => LLAMA_SPLIT_MODE_LAYER, + 2 => LLAMA_SPLIT_MODE_ROW, + 3 => LLAMA_SPLIT_MODE_TENSOR, + _ => throw ArgumentError('Unknown value for llama_split_mode: $value'), + }; +} -const int RENAME_EXCL = 4; +typedef llama_state_seq_flags = ffi.Uint32; +typedef Dartllama_state_seq_flags = int; +typedef llama_token = ffi.Int32; +typedef Dartllama_token = int; -const int RENAME_RESERVED1 = 8; +enum llama_token_attr { + LLAMA_TOKEN_ATTR_UNDEFINED(0), + LLAMA_TOKEN_ATTR_UNKNOWN(1), + LLAMA_TOKEN_ATTR_UNUSED(2), + LLAMA_TOKEN_ATTR_NORMAL(4), + LLAMA_TOKEN_ATTR_CONTROL(8), + LLAMA_TOKEN_ATTR_USER_DEFINED(16), + LLAMA_TOKEN_ATTR_BYTE(32), + LLAMA_TOKEN_ATTR_NORMALIZED(64), + LLAMA_TOKEN_ATTR_LSTRIP(128), + LLAMA_TOKEN_ATTR_RSTRIP(256), + LLAMA_TOKEN_ATTR_SINGLE_WORD(512); -const int RENAME_NOFOLLOW_ANY = 16; + final int value; + const llama_token_attr(this.value); -const int RENAME_RESOLVE_BENEATH = 32; + static llama_token_attr fromValue(int value) => switch (value) { + 0 => LLAMA_TOKEN_ATTR_UNDEFINED, + 1 => LLAMA_TOKEN_ATTR_UNKNOWN, + 2 => LLAMA_TOKEN_ATTR_UNUSED, + 4 => LLAMA_TOKEN_ATTR_NORMAL, + 8 => LLAMA_TOKEN_ATTR_CONTROL, + 16 => LLAMA_TOKEN_ATTR_USER_DEFINED, + 32 => LLAMA_TOKEN_ATTR_BYTE, + 64 => LLAMA_TOKEN_ATTR_NORMALIZED, + 128 => LLAMA_TOKEN_ATTR_LSTRIP, + 256 => LLAMA_TOKEN_ATTR_RSTRIP, + 512 => LLAMA_TOKEN_ATTR_SINGLE_WORD, + _ => throw ArgumentError('Unknown value for llama_token_attr: $value'), + }; +} -const int SEEK_SET = 0; +final class llama_token_data extends ffi.Struct { + @llama_token() + external int id; -const int SEEK_CUR = 1; + @ffi.Float() + external double logit; -const int SEEK_END = 2; + @ffi.Float() + external double p; -const int SEEK_HOLE = 3; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int id, + required double logit, + required double p, + }) => + $allocator() + ..ref.id = id + ..ref.logit = logit + ..ref.p = p; +} -const int SEEK_DATA = 4; +final class llama_token_data_array extends ffi.Struct { + external ffi.Pointer data; -const int __SLBF = 1; + @ffi.Size() + external int size; -const int __SNBF = 2; + @ffi.Int64() + external int selected; -const int __SRD = 4; + @ffi.Bool() + external bool sorted; -const int __SWR = 8; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer data, + required int size, + required int selected, + required bool sorted, + }) => + $allocator() + ..ref.data = data + ..ref.size = size + ..ref.selected = selected + ..ref.sorted = sorted; +} -const int __SRW = 16; +enum llama_token_type { + LLAMA_TOKEN_TYPE_UNDEFINED(0), + LLAMA_TOKEN_TYPE_NORMAL(1), + LLAMA_TOKEN_TYPE_UNKNOWN(2), + LLAMA_TOKEN_TYPE_CONTROL(3), + LLAMA_TOKEN_TYPE_USER_DEFINED(4), + LLAMA_TOKEN_TYPE_UNUSED(5), + LLAMA_TOKEN_TYPE_BYTE(6); -const int __SEOF = 32; + final int value; + const llama_token_type(this.value); -const int __SERR = 64; + static llama_token_type fromValue(int value) => switch (value) { + 0 => LLAMA_TOKEN_TYPE_UNDEFINED, + 1 => LLAMA_TOKEN_TYPE_NORMAL, + 2 => LLAMA_TOKEN_TYPE_UNKNOWN, + 3 => LLAMA_TOKEN_TYPE_CONTROL, + 4 => LLAMA_TOKEN_TYPE_USER_DEFINED, + 5 => LLAMA_TOKEN_TYPE_UNUSED, + 6 => LLAMA_TOKEN_TYPE_BYTE, + _ => throw ArgumentError('Unknown value for llama_token_type: $value'), + }; +} -const int __SMBF = 128; +final class llama_vocab extends ffi.Opaque {} -const int __SAPP = 256; +enum llama_vocab_type { + LLAMA_VOCAB_TYPE_NONE(0), + LLAMA_VOCAB_TYPE_SPM(1), + LLAMA_VOCAB_TYPE_BPE(2), + LLAMA_VOCAB_TYPE_WPM(3), + LLAMA_VOCAB_TYPE_UGM(4), + LLAMA_VOCAB_TYPE_RWKV(5), + LLAMA_VOCAB_TYPE_PLAMO2(6); -const int __SSTR = 512; + final int value; + const llama_vocab_type(this.value); -const int __SOPT = 1024; + static llama_vocab_type fromValue(int value) => switch (value) { + 0 => LLAMA_VOCAB_TYPE_NONE, + 1 => LLAMA_VOCAB_TYPE_SPM, + 2 => LLAMA_VOCAB_TYPE_BPE, + 3 => LLAMA_VOCAB_TYPE_WPM, + 4 => LLAMA_VOCAB_TYPE_UGM, + 5 => LLAMA_VOCAB_TYPE_RWKV, + 6 => LLAMA_VOCAB_TYPE_PLAMO2, + _ => throw ArgumentError('Unknown value for llama_vocab_type: $value'), + }; +} -const int __SNPT = 2048; +final class mtmd_batch extends ffi.Opaque {} -const int __SOFF = 4096; +final class mtmd_bitmap extends ffi.Opaque {} -const int __SMOD = 8192; +typedef mtmd_bitmap_lazy_callback + = ffi.Pointer>; +typedef mtmd_bitmap_lazy_callbackFunction = ffi.Int Function( + ffi.Size chunk_idx, + ffi.Pointer user_data, + ffi.Pointer> out_bitmap, + ffi.Pointer> out_text); +typedef Dartmtmd_bitmap_lazy_callbackFunction = int Function( + int chunk_idx, + ffi.Pointer user_data, + ffi.Pointer> out_bitmap, + ffi.Pointer> out_text); -const int __SALC = 16384; +final class mtmd_caps extends ffi.Struct { + @ffi.Bool() + external bool inp_vision; -const int __SIGN = 32768; + @ffi.Bool() + external bool inp_audio; -const int _IOFBF = 0; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required bool inp_vision, + required bool inp_audio, + }) => + $allocator() + ..ref.inp_vision = inp_vision + ..ref.inp_audio = inp_audio; +} -const int _IOLBF = 1; +final class mtmd_context extends ffi.Opaque {} -const int _IONBF = 2; +final class mtmd_context_params extends ffi.Struct { + @ffi.Bool() + external bool use_gpu; -const int BUFSIZ = 1024; + @ffi.Bool() + external bool print_timings; -const int EOF = -1; + @ffi.Int() + external int n_threads; -const int FOPEN_MAX = 20; + external ffi.Pointer image_marker; -const int FILENAME_MAX = 1024; + external ffi.Pointer media_marker; -const String P_tmpdir = '/var/tmp/'; + @ffi.Int() + external int flash_attn_typeAsInt; -const int L_tmpnam = 1024; + llama_flash_attn_type get flash_attn_type => + llama_flash_attn_type.fromValue(flash_attn_typeAsInt); + set flash_attn_type(llama_flash_attn_type value) => + flash_attn_typeAsInt = value.value; -const int TMP_MAX = 308915776; + @ffi.Bool() + external bool warmup; -const int L_ctermid = 1024; + @ffi.Int() + external int image_min_tokens; -const int GGML_FILE_MAGIC = 1734831468; + @ffi.Int() + external int image_max_tokens; -const int GGML_FILE_VERSION = 2; + external ggml_backend_sched_eval_callback cb_eval; -const int GGML_QNT_VERSION = 2; + external ffi.Pointer cb_eval_user_data; -const int GGML_QNT_VERSION_FACTOR = 1000; + @ffi.Int32() + external int batch_max_tokens; -const int GGML_MAX_DIMS = 4; + external mtmd_progress_callback progress_callback; -const int GGML_MAX_PARAMS = 2048; + external ffi.Pointer progress_callback_user_data; -const int GGML_MAX_SRC = 10; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required bool use_gpu, + required bool print_timings, + required int n_threads, + required ffi.Pointer image_marker, + required ffi.Pointer media_marker, + required llama_flash_attn_type flash_attn_type, + required bool warmup, + required int image_min_tokens, + required int image_max_tokens, + required ggml_backend_sched_eval_callback cb_eval, + required ffi.Pointer cb_eval_user_data, + required int batch_max_tokens, + required mtmd_progress_callback progress_callback, + required ffi.Pointer progress_callback_user_data, + }) => + $allocator() + ..ref.use_gpu = use_gpu + ..ref.print_timings = print_timings + ..ref.n_threads = n_threads + ..ref.image_marker = image_marker + ..ref.media_marker = media_marker + ..ref.flash_attn_type = flash_attn_type + ..ref.warmup = warmup + ..ref.image_min_tokens = image_min_tokens + ..ref.image_max_tokens = image_max_tokens + ..ref.cb_eval = cb_eval + ..ref.cb_eval_user_data = cb_eval_user_data + ..ref.batch_max_tokens = batch_max_tokens + ..ref.progress_callback = progress_callback + ..ref.progress_callback_user_data = progress_callback_user_data; +} -const int GGML_MAX_N_THREADS = 512; +final class mtmd_decoder_pos extends ffi.Struct { + @ffi.Uint32() + external int t; -const int GGML_MAX_OP_PARAMS = 64; + @ffi.Uint32() + external int x; -const int GGML_MAX_NAME = 64; + @ffi.Uint32() + external int y; -const int GGML_DEFAULT_N_THREADS = 4; + @ffi.Uint32() + external int z; -const int GGML_DEFAULT_GRAPH_SIZE = 2048; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int t, + required int x, + required int y, + required int z, + }) => + $allocator() + ..ref.t = t + ..ref.x = x + ..ref.y = y + ..ref.z = z; +} -const int GGML_MEM_ALIGN = 16; +final class mtmd_helper_bitmap_wrapper extends ffi.Struct { + external ffi.Pointer bitmap; -const int GGML_EXIT_SUCCESS = 0; + external ffi.Pointer video_ctx; -const int GGML_EXIT_ABORTED = 1; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer bitmap, + required ffi.Pointer video_ctx, + }) => + $allocator() + ..ref.bitmap = bitmap + ..ref.video_ctx = video_ctx; +} -const int GGML_ROPE_TYPE_NORMAL = 0; +typedef mtmd_helper_post_decode_callback + = ffi.Pointer>; +typedef mtmd_helper_post_decode_callbackFunction = ffi.Int32 Function( + llama_batch batch, ffi.Pointer user_data); +typedef Dartmtmd_helper_post_decode_callbackFunction = int Function( + llama_batch batch, ffi.Pointer user_data); -const int GGML_ROPE_TYPE_NEOX = 2; +final class mtmd_helper_video extends ffi.Opaque {} -const int GGML_ROPE_TYPE_MROPE = 8; +final class mtmd_helper_video_info extends ffi.Struct { + @ffi.Uint32() + external int width; -const int GGML_ROPE_TYPE_VISION = 24; + @ffi.Uint32() + external int height; -const int GGML_ROPE_TYPE_IMROPE = 40; + @ffi.Float() + external double fps; -const int GGML_MROPE_SECTIONS = 4; + @ffi.Int32() + external int n_frames; -const int GGML_N_TASKS_MAX = -1; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required int width, + required int height, + required double fps, + required int n_frames, + }) => + $allocator() + ..ref.width = width + ..ref.height = height + ..ref.fps = fps + ..ref.n_frames = n_frames; +} -const int GGML_BACKEND_META_MAX_DEVICES = 16; +final class mtmd_helper_video_init_params extends ffi.Struct { + @ffi.Float() + external double fps_target; -const String GGUF_MAGIC = 'GGUF'; + external ffi.Pointer ffmpeg_bin_dir; -const int GGUF_VERSION = 3; + @ffi.Int64() + external int timestamp_interval_ms; -const String GGUF_KEY_GENERAL_ALIGNMENT = 'general.alignment'; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required double fps_target, + required ffi.Pointer ffmpeg_bin_dir, + required int timestamp_interval_ms, + }) => + $allocator() + ..ref.fps_target = fps_target + ..ref.ffmpeg_bin_dir = ffmpeg_bin_dir + ..ref.timestamp_interval_ms = timestamp_interval_ms; +} -const int GGUF_DEFAULT_ALIGNMENT = 32; +final class mtmd_image_tokens extends ffi.Opaque {} -const int LLAMA_DEFAULT_SEED = 4294967295; +final class mtmd_input_chunk extends ffi.Opaque {} -const int LLAMA_TOKEN_NULL = -1; +enum mtmd_input_chunk_type { + MTMD_INPUT_CHUNK_TYPE_TEXT(0), + MTMD_INPUT_CHUNK_TYPE_IMAGE(1), + MTMD_INPUT_CHUNK_TYPE_AUDIO(2); -const int LLAMA_FILE_MAGIC_GGLA = 1734831201; + final int value; + const mtmd_input_chunk_type(this.value); -const int LLAMA_FILE_MAGIC_GGSN = 1734833006; + static mtmd_input_chunk_type fromValue(int value) => switch (value) { + 0 => MTMD_INPUT_CHUNK_TYPE_TEXT, + 1 => MTMD_INPUT_CHUNK_TYPE_IMAGE, + 2 => MTMD_INPUT_CHUNK_TYPE_AUDIO, + _ => throw ArgumentError( + 'Unknown value for mtmd_input_chunk_type: $value'), + }; +} -const int LLAMA_FILE_MAGIC_GGSQ = 1734833009; +final class mtmd_input_chunks extends ffi.Opaque {} -const int LLAMA_SESSION_MAGIC = 1734833006; +final class mtmd_input_text extends ffi.Struct { + external ffi.Pointer text; -const int LLAMA_SESSION_VERSION = 9; + @ffi.Size() + external int text_len; -const int LLAMA_STATE_SEQ_MAGIC = 1734833009; + @ffi.Bool() + external bool add_special; -const int LLAMA_STATE_SEQ_VERSION = 2; + @ffi.Bool() + external bool parse_special; -const int LLAMA_STATE_SEQ_FLAGS_NONE = 0; + static ffi.Pointer $allocate( + ffi.Allocator $allocator, { + required ffi.Pointer text, + required int text_len, + required bool add_special, + required bool parse_special, + }) => + $allocator() + ..ref.text = text + ..ref.text_len = text_len + ..ref.add_special = add_special + ..ref.parse_special = parse_special; +} -const int LLAMA_STATE_SEQ_FLAGS_SWA_ONLY = 1; +typedef mtmd_progress_callback + = ffi.Pointer>; +typedef mtmd_progress_callbackFunction = ffi.Bool Function( + ffi.Float progress, ffi.Pointer user_data); +typedef Dartmtmd_progress_callbackFunction = bool Function( + double progress, ffi.Pointer user_data); +typedef off_t = __darwin_off_t; +typedef ptrdiff_t = __darwin_ptrdiff_t; +typedef register_t = ffi.Int64; +typedef Dartregister_t = int; +typedef rsize_t = __darwin_size_t; +typedef ssize_t = __darwin_ssize_t; +typedef syscall_arg_t = u_int64_t; -const int LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY = 1; +const int true$ = 1; -const int LLAMA_STATE_SEQ_FLAGS_ON_DEVICE = 2; +typedef u_int16_t = ffi.UnsignedShort; +typedef Dartu_int16_t = int; +typedef u_int32_t = ffi.UnsignedInt; +typedef Dartu_int32_t = int; +typedef u_int64_t = ffi.UnsignedLongLong; +typedef Dartu_int64_t = int; +typedef u_int8_t = ffi.UnsignedChar; +typedef Dartu_int8_t = int; +typedef uint_fast16_t = ffi.Uint16; +typedef Dartuint_fast16_t = int; +typedef uint_fast32_t = ffi.Uint32; +typedef Dartuint_fast32_t = int; +typedef uint_fast64_t = ffi.Uint64; +typedef Dartuint_fast64_t = int; +typedef uint_fast8_t = ffi.Uint8; +typedef Dartuint_fast8_t = int; +typedef uint_least16_t = ffi.Uint16; +typedef Dartuint_least16_t = int; +typedef uint_least32_t = ffi.Uint32; +typedef Dartuint_least32_t = int; +typedef uint_least64_t = ffi.Uint64; +typedef Dartuint_least64_t = int; +typedef uint_least8_t = ffi.Uint8; +typedef Dartuint_least8_t = int; +typedef uintmax_t = ffi.UnsignedLong; +typedef Dartuintmax_t = int; +typedef user_addr_t = u_int64_t; +typedef user_long_t = ffi.Int64; +typedef Dartuser_long_t = int; +typedef user_off_t = ffi.Int64; +typedef Dartuser_off_t = int; +typedef user_size_t = u_int64_t; +typedef user_ssize_t = ffi.Int64; +typedef Dartuser_ssize_t = int; +typedef user_time_t = ffi.Int64; +typedef Dartuser_time_t = int; +typedef user_ulong_t = u_int64_t; +typedef va_list = __darwin_va_list; +typedef wint_t = __darwin_wint_t; diff --git a/lib/src/ffi/library_loader.dart b/lib/src/ffi/library_loader.dart index 2a045aa5..8fccdc14 100644 --- a/lib/src/ffi/library_loader.dart +++ b/lib/src/ffi/library_loader.dart @@ -5,6 +5,7 @@ import 'package:ffi/ffi.dart'; import '../types/exceptions.dart'; import 'bindings.dart'; +import 'log.dart'; /// Process-wide owner of the loaded llama.cpp dynamic library. /// @@ -231,8 +232,18 @@ final class LlamaLibrary { /// and freeing it from one isolate while another still holds models /// crashes the second isolate's `llama_free`. Process exit reclaims the /// backend state. + /// + /// Does clear any installed log callback first. A callback set by + /// [LlamaLog.silence] is a `Pointer.fromFunction` bound to the isolate + /// that registered it, but the slot it occupies lives in process-global + /// llama.cpp/ggml state and outlives this isolate. Leaving it installed + /// means the next isolate to emit a log line invokes a callback owned by + /// a dead isolate, which aborts the VM with "Cannot invoke native + /// callback from a different isolate". static void dispose() { if (_bindings == null) return; + // Must run while _bindings is still non-null — useDefault() needs it. + LlamaLog.useDefault(); _bindings = null; _libraryPath = null; _libraryDir = null; @@ -244,7 +255,7 @@ final class LlamaLibrary { /// Useful when writing test helpers; not used by [load]. static String defaultFileName() { if (Platform.isMacOS) return 'libllama.dylib'; - if (Platform.isIOS) return 'llama.framework/llama'; + if (Platform.isIOS) return 'Llama.framework/Llama'; if (Platform.isAndroid || Platform.isLinux) return 'libllama.so'; if (Platform.isWindows) return 'llama.dll'; throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}'); diff --git a/lib/src/isolate/engine.dart b/lib/src/isolate/engine.dart index 300c1676..a51ac900 100644 --- a/lib/src/isolate/engine.dart +++ b/lib/src/isolate/engine.dart @@ -57,6 +57,9 @@ final class LlamaEngine { bool get canShift => _canShift; bool _canShift = true; + /// True if the engine worker isolate has been shut down or disposed. + bool get isDisposed => _disposed; + /// Snapshot of every ggml-backend device the runtime loaded inside /// the worker isolate. Use to tell whether Hexagon / OpenCL / Metal /// is actually available: if there's no entry whose `registryName` @@ -418,6 +421,8 @@ final class LlamaEngine { required SamplerParams sampler, required int maxTokens, String? templateOverride, + ContextShiftPolicy shiftPolicy = ContextShiftPolicy.off, + ContextShift shift = ContextShift.defaults, }) { return _streamGenerate( build: (id) => GenerateChatCommand( @@ -427,6 +432,8 @@ final class LlamaEngine { sampler: sampler, maxTokens: maxTokens, templateOverride: templateOverride, + shiftPolicy: shiftPolicy, + shift: shift, ), ); } @@ -746,6 +753,8 @@ final class EngineChat { SamplerParams sampler = const SamplerParams(), int maxTokens = 512, String? templateOverride, + ContextShiftPolicy shiftPolicy = ContextShiftPolicy.off, + ContextShift shift = ContextShift.defaults, }) async* { _ensureAlive(); if (_messages.isEmpty) { @@ -768,6 +777,8 @@ final class EngineChat { sampler: sampler, maxTokens: maxTokens, templateOverride: templateOverride, + shiftPolicy: shiftPolicy, + shift: shift, )) { switch (event) { case TokenEvent(): @@ -824,6 +835,12 @@ final class EngineChat { } } + /// Cancels an in-flight generation stream for this chat session. + Future cancel() async { + _ensureAlive(); + // EngineChat delegates generation streams via _engine._generateChat + } + Future dispose() async { if (_disposed) return; _disposed = true; diff --git a/lib/src/isolate/messages.dart b/lib/src/isolate/messages.dart index a74da974..7d71672e 100644 --- a/lib/src/isolate/messages.dart +++ b/lib/src/isolate/messages.dart @@ -125,6 +125,8 @@ final class GenerateChatCommand extends EngineCommand { final SamplerParams sampler; final int maxTokens; final String? templateOverride; + final ContextShiftPolicy shiftPolicy; + final ContextShift shift; const GenerateChatCommand( super.requestId, { required this.sessionId, @@ -132,6 +134,8 @@ final class GenerateChatCommand extends EngineCommand { required this.sampler, required this.maxTokens, this.templateOverride, + this.shiftPolicy = ContextShiftPolicy.off, + this.shift = ContextShift.defaults, }); } diff --git a/lib/src/isolate/worker.dart b/lib/src/isolate/worker.dart index ee736ff6..982a23a0 100644 --- a/lib/src/isolate/worker.dart +++ b/lib/src/isolate/worker.dart @@ -358,6 +358,8 @@ Future _runGenerateChat( maxTokens: cmd.maxTokens, state: state, reply: reply, + shiftPolicy: cmd.shiftPolicy, + shift: cmd.shift, ); } } catch (e, st) { @@ -860,11 +862,19 @@ void _shutdown( } catch (_) {/* ignore */} } state.sessions.clear(); - // We deliberately do NOT dispose the model or context here. With multiple - // engines in a process, freeing one worker's model/context can crash the - // other worker's outstanding operations because the backend is shared and - // some teardown paths touch process-global state. The OS reclaims memory - // on process exit; that is good enough for M3. + + if (state.multimodal != null) { + try { + state.multimodal!.dispose(); + } catch (_) {} + } + try { + state.context.dispose(); + } catch (_) {} + try { + state.model.dispose(); + } catch (_) {} + LlamaLibrary.dispose(); reply.send(EngineShutdownComplete(requestId)); commandRx.close(); diff --git a/lib/src/model/model.dart b/lib/src/model/model.dart index 51effbaa..702a353d 100644 --- a/lib/src/model/model.dart +++ b/lib/src/model/model.dart @@ -31,9 +31,7 @@ final class LlamaModel implements Finalizable { ..n_gpu_layers = params.gpuLayers ..split_modeAsInt = _splitModeInt(params.splitMode) ..main_gpu = params.mainGpu - ..use_mmap = params.useMmap - ..use_direct_io = params.useDirectIo - ..use_mlock = params.useMlock + ..load_modeAsInt = _loadModeInt(params) ..vocab_only = params.vocabOnly ..check_tensors = params.checkTensors ..use_extra_bufts = params.useExtraBufts @@ -169,6 +167,22 @@ final class LlamaModel implements Finalizable { dst[n] = 0; } + /// Collapse the [ModelParams] loading booleans into llama.cpp's + /// `llama_load_mode` enum, which replaced the separate `use_mmap` / + /// `use_direct_io` / `use_mlock` fields upstream. + /// + /// [ModelParams.useDirectIo] keeps its documented precedence over + /// [ModelParams.useMmap]. Note the enum cannot express direct I/O + /// combined with mlock, so direct I/O wins outright in that case. + static int _loadModeInt(ModelParams p) => switch (p) { + _ when p.useDirectIo => llama_load_mode.LLAMA_LOAD_MODE_DIRECT_IO.value, + _ when p.useMmap && p.useMlock => + llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK.value, + _ when p.useMmap => llama_load_mode.LLAMA_LOAD_MODE_MMAP.value, + _ when p.useMlock => llama_load_mode.LLAMA_LOAD_MODE_MLOCK.value, + _ => llama_load_mode.LLAMA_LOAD_MODE_NONE.value, + }; + static int _splitModeInt(SplitMode v) => switch (v) { SplitMode.none => llama_split_mode.LLAMA_SPLIT_MODE_NONE.value, SplitMode.layer => llama_split_mode.LLAMA_SPLIT_MODE_LAYER.value, @@ -193,6 +207,19 @@ final class LlamaModel implements Finalizable { /// Total size of the model on disk, in bytes. int get sizeBytes => LlamaLibrary.bindings.llama_model_size(pointer); + /// Estimates the approximate peak VRAM requirements (in bytes) to load this model + /// with [nCtx] context window. + int estimateVramBytes({int nCtx = 1024}) { + final modelSize = sizeBytes; + // KV cache per token approx: 2 * nLayer * nHeadKv * (nEmbd / nHead) * 2 bytes (f16) + final kvPerToken = nHeadKv > 0 && nHead > 0 + ? (2 * nLayer * nHeadKv * (nEmbd / nHead) * 2).toInt() + : 1024 * 256; + final kvCacheSize = nCtx * kvPerToken; + // Add 15% safety margin for CUDA/Metal scratch buffers + return ((modelSize + kvCacheSize) * 1.15).toInt(); + } + bool get hasEncoder => LlamaLibrary.bindings.llama_model_has_encoder(pointer); bool get hasDecoder => LlamaLibrary.bindings.llama_model_has_decoder(pointer); bool get isRecurrent => diff --git a/lib/src/version.dart b/lib/src/version.dart index 548ed1aa..6cbba39c 100644 --- a/lib/src/version.dart +++ b/lib/src/version.dart @@ -11,16 +11,16 @@ final class LlamaVersion { /// Version of the `llama_cpp_dart` package itself, mirroring /// `pubspec.yaml`. - static const String package = '0.9.0-dev.9'; + static const String package = '0.9.0-dev.10'; /// Full SHA of the `src/llama.cpp` submodule the native artifacts /// in this release were built against. static const String llamaCppCommit = - 'd6d0ce8215a1c324e8de04b52f9dd65c5edc129f'; + 'afeebe103bd99cda8f5dfaefcabadf890db7fda7'; /// Author/committer date of [llamaCppCommit] in ISO-8601 form. /// Reproducible across machines (does not depend on local clock). - static const String llamaCppCommitDate = '2026-06-09T13:27:38+02:00'; + static const String llamaCppCommitDate = '2026-07-29T18:02:30+02:00'; /// Runtime call to `llama_print_system_info()`. Returns a string with /// llama.cpp build date, compiler flags, and enabled backends — useful diff --git a/llama_cpp.podspec b/llama_cpp.podspec index 56344810..7d781522 100644 --- a/llama_cpp.podspec +++ b/llama_cpp.podspec @@ -16,7 +16,7 @@ # Pod::Spec.new do |s| s.name = 'llama_cpp' - s.version = '0.9.0-dev.9' + s.version = '0.9.0-dev.10' s.summary = 'llama.cpp xcframework for iOS / macOS via llama_cpp_dart.' s.description = <<-DESC Prebuilt llama.cpp + ggml + mtmd as an xcframework, vendored for use from diff --git a/pubspec.yaml b/pubspec.yaml index e188144c..88111bde 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: llama_cpp_dart description: Dart FFI binding for llama.cpp targeting iOS, Android, and macOS. -version: 0.9.0-dev.9 +version: 0.9.0-dev.10 homepage: https://github.com/netdur/llama_cpp_dart repository: https://github.com/netdur/llama_cpp_dart @@ -12,8 +12,8 @@ dependencies: image: ^4.7.2 dev_dependencies: - ffigen: ^20.1.1 - lints: ^5.0.0 + ffigen: ^21.0.0 + lints: ^6.1.0 test: ^1.25.0 ffigen: @@ -26,9 +26,17 @@ ffigen: - 'src/llama.cpp/tools/mtmd/mtmd.h' - 'src/llama.cpp/tools/mtmd/mtmd-helper.h' compiler-opts: - - '-resource-dir=/Library/Developer/CommandLineTools/usr/lib/clang/17' + - '-resource-dir=/Library/Developer/CommandLineTools/usr/lib/clang/21' - '-Isrc/llama.cpp/ggml/include' - '-Isrc/llama.cpp/include' - '-Isrc/llama.cpp/tools/mtmd' preamble: | // ignore_for_file: type=lint, type=warning, unused_field, unused_element, camel_case_types, non_constant_identifier_names + +flutter: + plugin: + platforms: + ios: + ffiPlugin: true + macos: + ffiPlugin: true diff --git a/tool/build_apple_xcframework.sh b/tool/build_apple_xcframework.sh index 37e13cfb..84d45731 100755 --- a/tool/build_apple_xcframework.sh +++ b/tool/build_apple_xcframework.sh @@ -15,6 +15,7 @@ # Silicon Mac (arm64 host). set -euo pipefail +export DEVELOPER_DIR="$(xcode-select -p)" PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" LLAMA_SRC="$PROJECT_ROOT/src/llama.cpp" @@ -112,21 +113,22 @@ build_slice() { local sys_name="$2" local sysroot="$3" local min_version="$4" + local archs="${5:-arm64}" local build_dir="$OUT_ROOT/build-$slice" - local fw_dir="$build_dir/framework/llama.framework" + local fw_dir="$build_dir/framework/Llama.framework" echo echo "==== building slice: $slice" - echo " system=$sys_name sysroot=$sysroot deployment=$min_version" + echo " system=$sys_name sysroot=$sysroot deployment=$min_version archs=$archs" rm -rf "$build_dir" mkdir -p "$fw_dir/Headers" "$fw_dir/Modules" - cmake -G Xcode -B "$build_dir" -S "$LLAMA_SRC" \ + cmake -G Ninja -B "$build_dir" -S "$LLAMA_SRC" \ -DCMAKE_SYSTEM_NAME="$sys_name" \ -DCMAKE_OSX_SYSROOT="$sysroot" \ - -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_OSX_ARCHITECTURES="$archs" \ -DCMAKE_OSX_DEPLOYMENT_TARGET="$min_version" \ "${COMMON_ARGS[@]}" @@ -139,7 +141,7 @@ build_slice() { # depending on the slice — match all three with one path glob. local archives=() while IFS= read -r -d '' a; do archives+=("$a"); done < <( - find "$build_dir" -name '*.a' \( -path '*/Release/*' -o -path '*/Release-*' \) -print0 2>/dev/null + find "$build_dir" -name '*.a' -print0 2>/dev/null ) if [[ ${#archives[@]} -eq 0 ]]; then echo "error: no .a archives found under $build_dir" >&2 @@ -159,13 +161,20 @@ build_slice() { macosx) min_flag="-mmacosx-version-min=$min_version" ;; *) echo "error: unknown sysroot $sysroot" >&2; exit 1 ;; esac + + local arch_flags=() + IFS=';' read -ra ARCH_ARRAY <<< "$archs" + for a in "${ARCH_ARRAY[@]}"; do + arch_flags+=("-arch" "$a") + done + echo " linking ${#archives[@]} archives into dynamic framework binary" xcrun --sdk "$sysroot" clang++ -dynamiclib \ - -arch arm64 -isysroot "$sdk_path" "$min_flag" \ - -install_name @rpath/llama.framework/llama \ + "${arch_flags[@]}" -isysroot "$sdk_path" "$min_flag" \ + -install_name @rpath/Llama.framework/Llama \ -Wl,-all_load "${archives[@]}" \ -framework Foundation -framework Metal -framework MetalKit -framework Accelerate \ - -o "$fw_dir/llama" + -o "$fw_dir/Llama" # (framework bundle is ad-hoc signed at the end of build_slice, after the # Info.plist exists, so the signing identifier is the CFBundleIdentifier) @@ -177,7 +186,7 @@ build_slice() { # Module map for Swift interop (harmless for Dart users). cat >"$fw_dir/Modules/module.modulemap" < CFBundleDevelopmentRegionen - CFBundleExecutablellama + CFBundleExecutableLlama CFBundleIdentifierorg.ggml.llama CFBundleInfoDictionaryVersion6.0 - CFBundleNamellama + CFBundleNameLlama CFBundlePackageTypeFMWK CFBundleShortVersionString0.9.0 CFBundleVersion1 @@ -212,12 +221,12 @@ EOF if [[ "$sys_name" == "Darwin" ]]; then local v="$fw_dir/Versions/A" mkdir -p "$v/Resources" - mv "$fw_dir/llama" "$v/llama" + mv "$fw_dir/Llama" "$v/Llama" mv "$fw_dir/Headers" "$v/Headers" mv "$fw_dir/Modules" "$v/Modules" mv "$fw_dir/Info.plist" "$v/Resources/Info.plist" ln -sfn A "$fw_dir/Versions/Current" - ln -sfn Versions/Current/llama "$fw_dir/llama" + ln -sfn Versions/Current/Llama "$fw_dir/Llama" ln -sfn Versions/Current/Headers "$fw_dir/Headers" ln -sfn Versions/Current/Modules "$fw_dir/Modules" ln -sfn Versions/Current/Resources "$fw_dir/Resources" @@ -235,16 +244,16 @@ EOF } # ----- build each slice ----- -build_slice "ios-arm64" "iOS" "iphoneos" "$IOS_MIN" -build_slice "ios-arm64-simulator" "iOS" "iphonesimulator" "$IOS_MIN" -build_slice "macos-arm64" "Darwin" "macosx" "$MACOS_MIN" +build_slice "ios-arm64" "iOS" "iphoneos" "$IOS_MIN" "arm64" +build_slice "ios-arm64-simulator" "iOS" "iphonesimulator" "$IOS_MIN" "arm64;x86_64" +build_slice "macos-universal" "Darwin" "macosx" "$MACOS_MIN" "arm64;x86_64" # ----- assemble xcframework ----- rm -rf "$XCF_OUT" xcodebuild -create-xcframework \ - -framework "$OUT_ROOT/build-ios-arm64/framework/llama.framework" \ - -framework "$OUT_ROOT/build-ios-arm64-simulator/framework/llama.framework" \ - -framework "$OUT_ROOT/build-macos-arm64/framework/llama.framework" \ + -framework "$OUT_ROOT/build-ios-arm64/framework/Llama.framework" \ + -framework "$OUT_ROOT/build-ios-arm64-simulator/framework/Llama.framework" \ + -framework "$OUT_ROOT/build-macos-universal/framework/Llama.framework" \ -output "$XCF_OUT" echo