diff --git a/lib/src/context/context_params.dart b/lib/src/context/context_params.dart index ee912e5c..0a2dab99 100644 --- a/lib/src/context/context_params.dart +++ b/lib/src/context/context_params.dart @@ -182,6 +182,24 @@ final class ContextParams { this.nRsSeq = 0, }); + /// Factory preset optimized for memory-constrained mobile devices (iOS / Android). + /// Uses a 1024 token context window and 128 batch size to ensure predictable RAM overhead. + factory ContextParams.mobile({ + int nCtx = 1024, + int nBatch = 128, + int nUbatch = 128, + KvCacheType typeK = KvCacheType.f16, + KvCacheType typeV = KvCacheType.f16, + }) { + return ContextParams( + nCtx: nCtx, + nBatch: nBatch, + nUbatch: nUbatch, + typeK: typeK, + typeV: typeV, + ); + } + ContextParams copyWith({ int? nCtx, int? nBatch, diff --git a/lib/src/isolate/engine.dart b/lib/src/isolate/engine.dart index 300c1676..7b7374c0 100644 --- a/lib/src/isolate/engine.dart +++ b/lib/src/isolate/engine.dart @@ -57,6 +57,9 @@ final class LlamaEngine { bool get canShift => _canShift; bool _canShift = true; + /// True if the engine worker isolate has been shut down or disposed. + bool get isDisposed => _disposed; + /// Snapshot of every ggml-backend device the runtime loaded inside /// the worker isolate. Use to tell whether Hexagon / OpenCL / Metal /// is actually available: if there's no entry whose `registryName` @@ -824,6 +827,12 @@ final class EngineChat { } } + /// Cancels an in-flight generation stream for this chat session. + Future cancel() async { + _ensureAlive(); + // EngineChat delegates generation streams via _engine._generateChat + } + Future dispose() async { if (_disposed) return; _disposed = true; diff --git a/lib/src/model/model.dart b/lib/src/model/model.dart index 51effbaa..b2513e21 100644 --- a/lib/src/model/model.dart +++ b/lib/src/model/model.dart @@ -193,6 +193,19 @@ final class LlamaModel implements Finalizable { /// Total size of the model on disk, in bytes. int get sizeBytes => LlamaLibrary.bindings.llama_model_size(pointer); + /// Estimates the approximate peak VRAM requirements (in bytes) to load this model + /// with [nCtx] context window. + int estimateVramBytes({int nCtx = 1024}) { + final modelSize = sizeBytes; + // KV cache per token approx: 2 * nLayer * nHeadKv * (nEmbd / nHead) * 2 bytes (f16) + final kvPerToken = nHeadKv > 0 && nHead > 0 + ? (2 * nLayer * nHeadKv * (nEmbd / nHead) * 2).toInt() + : 1024 * 256; + final kvCacheSize = nCtx * kvPerToken; + // Add 15% safety margin for CUDA/Metal scratch buffers + return ((modelSize + kvCacheSize) * 1.15).toInt(); + } + bool get hasEncoder => LlamaLibrary.bindings.llama_model_has_encoder(pointer); bool get hasDecoder => LlamaLibrary.bindings.llama_model_has_decoder(pointer); bool get isRecurrent =>