Skip to content

Add LTX Ingredients reference conditioning #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aegioscy wants to merge 3 commits into
base: 2026-07-03
Choose a base branch
from
Open

Add LTX Ingredients reference conditioning #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
781b396
feat(ltx): add Ingredients reference conditioning
aegioscy Jul 27, 2026
401fc93
fix(ltx): align Ingredients reference positions
aegioscy Jul 28, 2026
146ede0
fix(ltx): align IC-LoRA inference workflow
aegioscy Jul 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion examples/common/common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1027,6 +1027,14 @@ ArgOptions SDGenerationParams::get_options() {
"--vace-strength",
"wan vace strength",
&vace_strength},
{"",
"--reference-attention-strength",
"LTX IC-LoRA reference conditioning strength in [0, 1] (default: 1.0)",
&reference_attention_strength},
{"",
"--reference-downscale-factor",
"LTX IC-LoRA reference image downscale factor (currently must be 1.0; default: 1.0)",
&reference_downscale_factor},
{"",
"--vae-tile-overlap",
"tile overlap for vae tiling, in fraction of tile size (default: 0.5)",
Expand Down Expand Up @@ -1366,7 +1374,7 @@ ArgOptions SDGenerationParams::get_options() {
on_high_noise_skip_layers_arg},
{"-r",
"--ref-image",
"reference image for Flux Kontext models (can be used multiple times)",
"reference image for Flux Kontext or LTX IC-LoRA video models (can be used multiple times)",
on_ref_image_arg},
{"",
"--cache-mode",
Expand Down Expand Up @@ -2141,6 +2149,17 @@ bool SDGenerationParams::validate(SDMode mode) {
return false;
}

if (mode == VID_GEN && !ref_image_paths.empty()) {
if (reference_attention_strength < 0.f || reference_attention_strength > 1.f) {
LOG_ERROR("error: reference attention strength must be in [0, 1]");
return false;
}
if (reference_downscale_factor != 1.f) {
LOG_ERROR("error: LTX IC-LoRA currently requires reference downscale factor to be 1");
return false;
}
}

if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) {
LOG_ERROR("error: shifted_timestep must be in range [0, 1000]");
return false;
Expand Down Expand Up @@ -2303,6 +2322,12 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
control_frame_views.push_back(frame.get());
}

ref_image_views.clear();
ref_image_views.reserve(ref_images.size());
for (auto& ref_image : ref_images) {
ref_image_views.push_back(ref_image.get());
}

sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data();
sample_params.guidance.slg.layer_count = skip_layers.size();
high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data();
Expand All @@ -2323,6 +2348,10 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
params.end_image = end_image.get();
params.control_frames = control_frame_views.empty() ? nullptr : control_frame_views.data();
params.control_frames_size = static_cast<int>(control_frame_views.size());
params.reference_images = ref_image_views.empty() ? nullptr : ref_image_views.data();
params.reference_images_count = static_cast<int>(ref_image_views.size());
params.reference_attention_strength = reference_attention_strength;
params.reference_downscale_factor = reference_downscale_factor;
params.width = get_resolved_width();
params.height = get_resolved_height();
params.sample_params = sample_params;
Expand Down Expand Up @@ -2415,6 +2444,8 @@ std::string SDGenerationParams::to_string() const {
<< " video_frames: " << video_frames << ",\n"
<< " fps: " << fps << ",\n"
<< " vace_strength: " << vace_strength << ",\n"
<< " reference_attention_strength: " << reference_attention_strength << ",\n"
<< " reference_downscale_factor: " << reference_downscale_factor << ",\n"
<< " strength: " << strength << ",\n"
<< " control_strength: " << control_strength << ",\n"
<< " seed: " << seed << ",\n"
Expand Down
2 changes: 2 additions & 0 deletions examples/common/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ struct SDGenerationParams {
int video_frames = 1;
int fps = 16;
float vace_strength = 1.f;
float reference_attention_strength = 1.f;
float reference_downscale_factor = 1.f;
sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
std::string extra_tiling_args;

Expand Down
11 changes: 11 additions & 0 deletions include/stable-diffusion.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,17 @@ typedef struct {
sd_tiling_params_t vae_tiling_params;
sd_cache_params_t cache;
sd_hires_params_t hires;
// LTX IC-LoRA reference conditioning. These fields are ignored for
// non-LTX video models. Each input is a reference sheet/image that is
// expanded into a static reference video before VAE encoding.
sd_image_t* reference_images;
int reference_images_count;
// [0, 1]. Zero disables the reference tokens; one preserves their full
// conditioning weight. Defaults are set by sd_vid_gen_params_init().
float reference_attention_strength;
// The current LTX Ingredients workflow requires 1.0. Values other than
// one are rejected until a spatial downscale implementation is added.
float reference_downscale_factor;
} sd_vid_gen_params_t;

typedef struct sd_ctx_t sd_ctx_t;
Expand Down
1 change: 1 addition & 0 deletions src/diffusion_model.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ struct LTXAVDiffusionExtra {
int audio_length = 0;
float frame_rate = 24.f;
const sd::Tensor<float>* video_positions = nullptr;
const std::vector<int>* skip_video_self_attention_blocks = nullptr;
};

using DiffusionExtraParams = std::variant<std::monostate,
Expand Down
85 changes: 54 additions & 31 deletions src/ltxv.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1071,7 +1071,8 @@ namespace LTXV {
ggml_tensor* a_cross_gate_timestep,
ggml_tensor* v_prompt_timestep,
ggml_tensor* a_prompt_timestep,
ggml_tensor* self_attention_mask = nullptr) {
ggml_tensor* self_attention_mask = nullptr,
bool skip_video_self_attention = false) {
auto attn1 = std::dynamic_pointer_cast<CrossAttention>(blocks["attn1"]);
auto audio_attn1 = std::dynamic_pointer_cast<CrossAttention>(blocks["audio_attn1"]);
auto attn2 = std::dynamic_pointer_cast<CrossAttention>(blocks["attn2"]);
Expand All @@ -1089,10 +1090,12 @@ namespace LTXV {
bool run_v2a = run_ax;

auto v_mods = get_ada_values(ctx, v_table, v_timestep, v_dim, cross_attention_adaln ? 9 : 6);
auto v_norm = rms_norm(ctx->ggml_ctx, vx);
v_norm = modulate(ctx->ggml_ctx, v_norm, v_mods[0], v_mods[1]);
auto v_sa = attn1->forward(ctx, v_norm, nullptr, self_attention_mask, v_pe);
vx = ggml_add(ctx->ggml_ctx, vx, apply_gate(ctx->ggml_ctx, v_sa, v_mods[2]));
if (!skip_video_self_attention) {
auto v_norm = rms_norm(ctx->ggml_ctx, vx);
v_norm = modulate(ctx->ggml_ctx, v_norm, v_mods[0], v_mods[1]);
auto v_sa = attn1->forward(ctx, v_norm, nullptr, self_attention_mask, v_pe);
vx = ggml_add(ctx->ggml_ctx, vx, apply_gate(ctx->ggml_ctx, v_sa, v_mods[2]));
}
auto v_txt = apply_text_cross_attention(ctx,
vx,
v_context,
Expand Down Expand Up @@ -1418,7 +1421,8 @@ namespace LTXV {
ggml_tensor* v_cross_pe,
ggml_tensor* a_cross_pe,
ggml_tensor* video_connector_pe,
ggml_tensor* audio_connector_pe) {
ggml_tensor* audio_connector_pe,
const std::vector<int>* skip_video_self_attention_blocks = nullptr) {
auto patchify_proj = std::dynamic_pointer_cast<Linear>(blocks["patchify_proj"]);
auto audio_patchify_proj = std::dynamic_pointer_cast<Linear>(blocks["audio_patchify_proj"]);
auto adaln_single = std::dynamic_pointer_cast<AdaLayerNormSingle>(blocks["adaln_single"]);
Expand Down Expand Up @@ -1493,26 +1497,33 @@ namespace LTXV {

for (int i = 0; i < cfg.num_layers; i++) {
auto block = std::dynamic_pointer_cast<BasicAVTransformerBlock>(blocks["transformer_blocks." + std::to_string(i)]);
auto out = block->forward(ctx,
vx,
ax,
v_context,
a_context,
nullptr,
v_timestep_mod,
a_timestep_mod,
v_pe,
a_pe,
v_cross_pe,
a_cross_pe,
av_ca_video_scale_shift_timestep,
av_ca_audio_scale_shift_timestep,
av_ca_a2v_gate_noise_timestep,
av_ca_v2a_gate_noise_timestep,
v_prompt_timestep_mod,
a_prompt_timestep_mod);
vx = out.first;
ax = out.second;
const bool skip_video_self_attention =
skip_video_self_attention_blocks != nullptr &&
std::find(skip_video_self_attention_blocks->begin(),
skip_video_self_attention_blocks->end(),
i) != skip_video_self_attention_blocks->end();
auto out = block->forward(ctx,
vx,
ax,
v_context,
a_context,
nullptr,
v_timestep_mod,
a_timestep_mod,
v_pe,
a_pe,
v_cross_pe,
a_cross_pe,
av_ca_video_scale_shift_timestep,
av_ca_audio_scale_shift_timestep,
av_ca_a2v_gate_noise_timestep,
av_ca_v2a_gate_noise_timestep,
v_prompt_timestep_mod,
a_prompt_timestep_mod,
nullptr,
skip_video_self_attention);
vx = out.first;
ax = out.second;
sd::ggml_graph_cut::mark_graph_cut(vx, "ltxav.transformer_blocks." + std::to_string(i), "vx");
sd::ggml_graph_cut::mark_graph_cut(ax, "ltxav.transformer_blocks." + std::to_string(i), "ax");
}
Expand Down Expand Up @@ -1744,7 +1755,8 @@ namespace LTXV {
const sd::Tensor<float>& audio_timesteps_tensor = {},
int audio_length = 0,
float frame_rate = 24.f,
const sd::Tensor<float>& video_positions_tensor = {}) {
const sd::Tensor<float>& video_positions_tensor = {},
const std::vector<int>* skip_video_self_attention_blocks = nullptr) {
auto split_inputs = split_av_latents(x_tensor, audio_length);
vx_input_cache = split_inputs.first;
if (!audio_x_tensor.empty()) {
Expand Down Expand Up @@ -1894,7 +1906,8 @@ namespace LTXV {
video_cross_pe,
audio_cross_pe,
video_connector_pe,
audio_connector_pe);
audio_connector_pe,
skip_video_self_attention_blocks);
auto out = merge_av_latents(compute_ctx, out_pair.first, out_pair.second);
ggml_build_forward_expand(gf, out);
return gf;
Expand All @@ -1908,9 +1921,18 @@ namespace LTXV {
const sd::Tensor<float>& audio_timesteps = {},
int audio_length = 0,
float frame_rate = 24.f,
const sd::Tensor<float>& video_positions = {}) {
const sd::Tensor<float>& video_positions = {},
const std::vector<int>* skip_video_self_attention_blocks = nullptr) {
auto get_graph = [&]() -> ggml_cgraph* {
return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions);
return build_graph(x,
timesteps,
context,
audio_x,
audio_timesteps,
audio_length,
frame_rate,
video_positions,
skip_video_self_attention_blocks);
};
auto out = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false), x.dim());
return out;
Expand All @@ -1929,7 +1951,8 @@ namespace LTXV {
tensor_or_empty(extra->audio_timesteps),
extra->audio_length,
extra->frame_rate,
tensor_or_empty(extra->video_positions));
tensor_or_empty(extra->video_positions),
extra->skip_video_self_attention_blocks);
}

void test(const std::string& x_path,
Expand Down
Loading
Loading