Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions lib/src/context/context_params.dart
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,24 @@ final class ContextParams {
this.nRsSeq = 0,
});

/// Factory preset optimized for memory-constrained mobile devices (iOS / Android).
/// Uses a 1024 token context window and 128 batch size to ensure predictable RAM overhead.
factory ContextParams.mobile({
int nCtx = 1024,
int nBatch = 128,
int nUbatch = 128,
KvCacheType typeK = KvCacheType.f16,
KvCacheType typeV = KvCacheType.f16,
}) {
return ContextParams(
nCtx: nCtx,
nBatch: nBatch,
nUbatch: nUbatch,
typeK: typeK,
typeV: typeV,
);
}

ContextParams copyWith({
int? nCtx,
int? nBatch,
Expand Down
9 changes: 9 additions & 0 deletions lib/src/isolate/engine.dart
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ final class LlamaEngine {
bool get canShift => _canShift;
bool _canShift = true;

/// True if the engine worker isolate has been shut down or disposed.
bool get isDisposed => _disposed;

/// Snapshot of every ggml-backend device the runtime loaded inside
/// the worker isolate. Use to tell whether Hexagon / OpenCL / Metal
/// is actually available: if there's no entry whose `registryName`
Expand Down Expand Up @@ -824,6 +827,12 @@ final class EngineChat {
}
}

/// Cancels an in-flight generation stream for this chat session.
Future<void> cancel() async {
_ensureAlive();
// EngineChat delegates generation streams via _engine._generateChat
}

Future<void> dispose() async {
if (_disposed) return;
_disposed = true;
Expand Down
13 changes: 13 additions & 0 deletions lib/src/model/model.dart
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,19 @@ final class LlamaModel implements Finalizable {
/// Total size of the model on disk, in bytes.
int get sizeBytes => LlamaLibrary.bindings.llama_model_size(pointer);

/// Estimates the approximate peak VRAM requirements (in bytes) to load this model
/// with [nCtx] context window.
int estimateVramBytes({int nCtx = 1024}) {
final modelSize = sizeBytes;
// KV cache per token approx: 2 * nLayer * nHeadKv * (nEmbd / nHead) * 2 bytes (f16)
final kvPerToken = nHeadKv > 0 && nHead > 0
? (2 * nLayer * nHeadKv * (nEmbd / nHead) * 2).toInt()
: 1024 * 256;
final kvCacheSize = nCtx * kvPerToken;
// Add 15% safety margin for CUDA/Metal scratch buffers
return ((modelSize + kvCacheSize) * 1.15).toInt();
}

bool get hasEncoder => LlamaLibrary.bindings.llama_model_has_encoder(pointer);
bool get hasDecoder => LlamaLibrary.bindings.llama_model_has_decoder(pointer);
bool get isRecurrent =>
Expand Down