From 781b396e8acae97e9e6bfb955611e4a51d4a43d0 Mon Sep 17 00:00:00 2001 From: aegioscy Date: Mon, 27 Jul 2026 10:06:51 +0200 Subject: [PATCH 1/3] feat(ltx): add Ingredients reference conditioning Expose static reference-sheet conditioning through the LTX video API and CLI so IC-LoRA adapters can preserve reference content during generation. --- examples/common/common.cpp | 33 +++++++++++- examples/common/common.h | 2 + include/stable-diffusion.h | 11 ++++ src/stable-diffusion.cpp | 102 ++++++++++++++++++++++++++++++++++++- 4 files changed, 146 insertions(+), 2 deletions(-) diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 9a22efe6a..88625c853 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1027,6 +1027,14 @@ ArgOptions SDGenerationParams::get_options() { "--vace-strength", "wan vace strength", &vace_strength}, + {"", + "--reference-attention-strength", + "LTX IC-LoRA reference conditioning strength in [0, 1] (default: 1.0)", + &reference_attention_strength}, + {"", + "--reference-downscale-factor", + "LTX IC-LoRA reference image downscale factor (currently must be 1.0; default: 1.0)", + &reference_downscale_factor}, {"", "--vae-tile-overlap", "tile overlap for vae tiling, in fraction of tile size (default: 0.5)", @@ -1366,7 +1374,7 @@ ArgOptions SDGenerationParams::get_options() { on_high_noise_skip_layers_arg}, {"-r", "--ref-image", - "reference image for Flux Kontext models (can be used multiple times)", + "reference image for Flux Kontext or LTX IC-LoRA video models (can be used multiple times)", on_ref_image_arg}, {"", "--cache-mode", @@ -2141,6 +2149,17 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } + if (mode == VID_GEN && !ref_image_paths.empty()) { + if (reference_attention_strength < 0.f || reference_attention_strength > 1.f) { + LOG_ERROR("error: reference attention strength must be in [0, 1]"); + return false; + } + if (reference_downscale_factor != 1.f) { + LOG_ERROR("error: LTX IC-LoRA currently requires reference downscale factor to be 1"); + return false; + } + } + if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) { LOG_ERROR("error: shifted_timestep must be in range [0, 1000]"); return false; @@ -2303,6 +2322,12 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { control_frame_views.push_back(frame.get()); } + ref_image_views.clear(); + ref_image_views.reserve(ref_images.size()); + for (auto& ref_image : ref_images) { + ref_image_views.push_back(ref_image.get()); + } + sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data(); sample_params.guidance.slg.layer_count = skip_layers.size(); high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data(); @@ -2323,6 +2348,10 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { params.end_image = end_image.get(); params.control_frames = control_frame_views.empty() ? nullptr : control_frame_views.data(); params.control_frames_size = static_cast(control_frame_views.size()); + params.reference_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); + params.reference_images_count = static_cast(ref_image_views.size()); + params.reference_attention_strength = reference_attention_strength; + params.reference_downscale_factor = reference_downscale_factor; params.width = get_resolved_width(); params.height = get_resolved_height(); params.sample_params = sample_params; @@ -2415,6 +2444,8 @@ std::string SDGenerationParams::to_string() const { << " video_frames: " << video_frames << ",\n" << " fps: " << fps << ",\n" << " vace_strength: " << vace_strength << ",\n" + << " reference_attention_strength: " << reference_attention_strength << ",\n" + << " reference_downscale_factor: " << reference_downscale_factor << ",\n" << " strength: " << strength << ",\n" << " control_strength: " << control_strength << ",\n" << " seed: " << seed << ",\n" diff --git a/examples/common/common.h b/examples/common/common.h index 999d23d52..41af8aad0 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -192,6 +192,8 @@ struct SDGenerationParams { int video_frames = 1; int fps = 16; float vace_strength = 1.f; + float reference_attention_strength = 1.f; + float reference_downscale_factor = 1.f; sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; std::string extra_tiling_args; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index f2b6f00fc..c62dfc846 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -404,6 +404,17 @@ typedef struct { sd_tiling_params_t vae_tiling_params; sd_cache_params_t cache; sd_hires_params_t hires; + // LTX IC-LoRA reference conditioning. These fields are ignored for + // non-LTX video models. Each input is a reference sheet/image that is + // expanded into a static reference video before VAE encoding. + sd_image_t* reference_images; + int reference_images_count; + // [0, 1]. Zero disables the reference tokens; one preserves their full + // conditioning weight. Defaults are set by sd_vid_gen_params_init(). + float reference_attention_strength; + // The current LTX Ingredients workflow requires 1.0. Values other than + // one are rejected until a spatial downscale implementation is added. + float reference_downscale_factor; } sd_vid_gen_params_t; typedef struct sd_ctx_t sd_ctx_t; diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index df84281b2..7fad365d0 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -3082,6 +3082,10 @@ void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params) { sd_vid_gen_params->fps = 16; sd_vid_gen_params->moe_boundary = 0.875f; sd_vid_gen_params->vace_strength = 1.f; + sd_vid_gen_params->reference_images = nullptr; + sd_vid_gen_params->reference_images_count = 0; + sd_vid_gen_params->reference_attention_strength = 1.f; + sd_vid_gen_params->reference_downscale_factor = 1.f; sd_vid_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; sd_vid_gen_params->hires.enabled = false; sd_vid_gen_params->hires.upscaler = SD_HIRES_UPSCALER_LATENT; @@ -3827,6 +3831,30 @@ static sd::Tensor make_ltxav_video_denoise_mask(const sd::Tensor& value); } +static sd::Tensor make_ltxav_static_reference_video(const sd::Tensor& image, int frames) { + if (image.empty() || image.dim() != 4 || frames <= 0) { + return {}; + } + + const int64_t width = image.shape()[0]; + const int64_t height = image.shape()[1]; + const int64_t channels = image.shape()[2]; + const int64_t batches = image.shape()[3]; + const int64_t plane = width * height; + sd::Tensor video({width, height, frames, channels, batches}); + + for (int64_t batch = 0; batch < batches; ++batch) { + for (int64_t channel = 0; channel < channels; ++channel) { + const float* source = image.data() + (batch * channels + channel) * plane; + for (int frame = 0; frame < frames; ++frame) { + float* destination = video.data() + ((batch * channels + channel) * frames + frame) * plane; + std::copy_n(source, plane, destination); + } + } + } + return video; +} + static sd::Tensor encode_ltxav_condition_image(sd_ctx_t* sd_ctx, const sd::Tensor& image, const char* name) { @@ -4812,6 +4840,7 @@ static std::optional prepare_video_generation_latents(sd sd::Tensor start_image; sd::Tensor end_image; + std::vector> reference_images; if (sd_vid_gen_params->init_image.data) { start_image = sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height); @@ -4820,10 +4849,30 @@ static std::optional prepare_video_generation_latents(sd if (sd_vid_gen_params->end_image.data) { end_image = sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height); } + if (sd_vid_gen_params->reference_images_count < 0 || + (sd_vid_gen_params->reference_images_count > 0 && sd_vid_gen_params->reference_images == nullptr)) { + LOG_ERROR("invalid LTX IC-LoRA reference image inputs"); + return std::nullopt; + } + for (int i = 0; i < sd_vid_gen_params->reference_images_count; ++i) { + const sd_image_t& image = sd_vid_gen_params->reference_images[i]; + if (image.data == nullptr) { + LOG_ERROR("LTX IC-LoRA reference image %d is empty", i); + return std::nullopt; + } + reference_images.push_back(sd_image_to_tensor(image, request->width, request->height)); + if (reference_images.back().empty()) { + LOG_ERROR("failed to prepare LTX IC-LoRA reference image %d", i); + return std::nullopt; + } + } if (sd_version_is_ltxav(sd_ctx->sd->version)) { latents.audio_length = get_ltxav_num_audio_latents(request->frames, request->fps); latents.audio_latent = make_ltxav_empty_audio_latent(latents.audio_length); + } else if (!reference_images.empty()) { + LOG_ERROR("IC-LoRA reference conditioning is supported only for LTX video models"); + return std::nullopt; } if (sd_version_is_ltxav(sd_ctx->sd->version)) { @@ -4832,7 +4881,24 @@ static std::optional prepare_video_generation_latents(sd return std::nullopt; } - if (!start_image.empty() || !end_image.empty()) { + if (!reference_images.empty() && + (!start_image.empty() || !end_image.empty())) { + LOG_ERROR("LTX IC-LoRA reference conditioning cannot be combined with init_image or end_image"); + return std::nullopt; + } + if (!reference_images.empty()) { + if (sd_vid_gen_params->reference_downscale_factor != 1.f) { + LOG_ERROR("LTX IC-LoRA currently requires reference_downscale_factor=1"); + return std::nullopt; + } + if (sd_vid_gen_params->reference_attention_strength < 0.f || + sd_vid_gen_params->reference_attention_strength > 1.f) { + LOG_ERROR("LTX IC-LoRA reference_attention_strength must be in [0, 1]"); + return std::nullopt; + } + } + + if (!start_image.empty() || !end_image.empty() || !reference_images.empty()) { if (sd_ctx->sd->vae_decode_only) { LOG_ERROR("LTXAV image conditioning requires VAE encoder weights; create the context with vae_decode_only=false"); return std::nullopt; @@ -4842,6 +4908,8 @@ static std::optional prepare_video_generation_latents(sd LOG_INFO("FLF2V"); } else if (!start_image.empty()) { LOG_INFO("IMG2VID"); + } else if (!reference_images.empty()) { + LOG_INFO("LTX IC-LoRA reference conditioning"); } else { LOG_INFO("END2VID"); } @@ -4919,6 +4987,38 @@ static std::optional prepare_video_generation_latents(sd } } + if (!reference_images.empty()) { + sd::Tensor reference_latents; + for (size_t i = 0; i < reference_images.size(); ++i) { + auto static_reference = + make_ltxav_static_reference_video(reference_images[i], request->frames); + auto reference_latent = sd_ctx->sd->encode_first_stage(static_reference); + if (reference_latent.empty()) { + LOG_ERROR("failed to encode LTX IC-LoRA reference image %zu", i); + return std::nullopt; + } + reference_latents = reference_latents.empty() + ? std::move(reference_latent) + : sd::ops::concat(reference_latents, reference_latent, 2); + } + if (!apply_video_condition_by_keyframe_index(reference_latents, + 0, + "IC-LoRA reference")) { + return std::nullopt; + } + const float reference_strength = + std::clamp(sd_vid_gen_params->reference_attention_strength, 0.f, 1.f); + const int64_t reference_frames = reference_latents.shape()[2]; + sd::ops::fill_slice(&latents.denoise_mask, + 2, + latents.denoise_mask.shape()[2] - reference_frames, + latents.denoise_mask.shape()[2], + 1.f - reference_strength); + LOG_INFO("LTX IC-LoRA conditioned on %zu static reference image(s), strength %.3f", + reference_images.size(), + reference_strength); + } + int64_t t2 = ggml_time_ms(); LOG_INFO("encode_first_stage completed, taking %" PRId64 " ms", t2 - t1); } From 401fc93993640ed889aadfc09355e1e7d125dbaf Mon Sep 17 00:00:00 2001 From: aegioscy Date: Tue, 28 Jul 2026 10:13:08 +0200 Subject: [PATCH 2/3] fix(ltx): align Ingredients reference positions Use causal temporal positions for appended reference tokens and reject unsupported multi-sheet conditioning. --- src/stable-diffusion.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 7fad365d0..2b2f4dc78 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -3717,8 +3717,12 @@ static sd::Tensor build_ltxv_video_positions(int64_t width, } for (int64_t t = 0; t < keyframe_latent_frames; t++) { - float t_start = static_cast(keyframe_frame_idx + t * temporal_scale); - float t_end = static_cast(keyframe_frame_idx + (t + 1) * temporal_scale); + const int64_t start_corner = keyframe_frame_idx + t; + const int64_t end_corner = keyframe_frame_idx + t + 1; + float t_start = ltxv_latent_corner_to_pixel_frame( + start_corner, temporal_scale, causal_temporal_positioning); + float t_end = ltxv_latent_corner_to_pixel_frame( + end_corner, temporal_scale, causal_temporal_positioning); if (keyframe_pixel_frames == 1) { t_end = t_start + 1.f; } @@ -4887,6 +4891,10 @@ static std::optional prepare_video_generation_latents(sd return std::nullopt; } if (!reference_images.empty()) { + if (reference_images.size() != 1) { + LOG_ERROR("LTX Ingredients requires exactly one composite reference sheet"); + return std::nullopt; + } if (sd_vid_gen_params->reference_downscale_factor != 1.f) { LOG_ERROR("LTX IC-LoRA currently requires reference_downscale_factor=1"); return std::nullopt; From 146ede0c888b5efa7db9521aefd4164a42651242 Mon Sep 17 00:00:00 2001 From: aegioscy Date: Tue, 28 Jul 2026 17:26:05 +0200 Subject: [PATCH 3/3] fix(ltx): align IC-LoRA inference workflow Match official reference-first conditioning and add video-only spatiotemporal guidance so Ingredients generation preserves reference identity and motion quality. --- src/diffusion_model.hpp | 1 + src/ltxv.hpp | 85 ++++++++++++++++++++------------ src/stable-diffusion.cpp | 104 +++++++++++++++++++++------------------ 3 files changed, 112 insertions(+), 78 deletions(-) diff --git a/src/diffusion_model.hpp b/src/diffusion_model.hpp index 80c115449..bde899e76 100644 --- a/src/diffusion_model.hpp +++ b/src/diffusion_model.hpp @@ -47,6 +47,7 @@ struct LTXAVDiffusionExtra { int audio_length = 0; float frame_rate = 24.f; const sd::Tensor* video_positions = nullptr; + const std::vector* skip_video_self_attention_blocks = nullptr; }; using DiffusionExtraParams = std::variant(blocks["attn1"]); auto audio_attn1 = std::dynamic_pointer_cast(blocks["audio_attn1"]); auto attn2 = std::dynamic_pointer_cast(blocks["attn2"]); @@ -1089,10 +1090,12 @@ namespace LTXV { bool run_v2a = run_ax; auto v_mods = get_ada_values(ctx, v_table, v_timestep, v_dim, cross_attention_adaln ? 9 : 6); - auto v_norm = rms_norm(ctx->ggml_ctx, vx); - v_norm = modulate(ctx->ggml_ctx, v_norm, v_mods[0], v_mods[1]); - auto v_sa = attn1->forward(ctx, v_norm, nullptr, self_attention_mask, v_pe); - vx = ggml_add(ctx->ggml_ctx, vx, apply_gate(ctx->ggml_ctx, v_sa, v_mods[2])); + if (!skip_video_self_attention) { + auto v_norm = rms_norm(ctx->ggml_ctx, vx); + v_norm = modulate(ctx->ggml_ctx, v_norm, v_mods[0], v_mods[1]); + auto v_sa = attn1->forward(ctx, v_norm, nullptr, self_attention_mask, v_pe); + vx = ggml_add(ctx->ggml_ctx, vx, apply_gate(ctx->ggml_ctx, v_sa, v_mods[2])); + } auto v_txt = apply_text_cross_attention(ctx, vx, v_context, @@ -1418,7 +1421,8 @@ namespace LTXV { ggml_tensor* v_cross_pe, ggml_tensor* a_cross_pe, ggml_tensor* video_connector_pe, - ggml_tensor* audio_connector_pe) { + ggml_tensor* audio_connector_pe, + const std::vector* skip_video_self_attention_blocks = nullptr) { auto patchify_proj = std::dynamic_pointer_cast(blocks["patchify_proj"]); auto audio_patchify_proj = std::dynamic_pointer_cast(blocks["audio_patchify_proj"]); auto adaln_single = std::dynamic_pointer_cast(blocks["adaln_single"]); @@ -1493,26 +1497,33 @@ namespace LTXV { for (int i = 0; i < cfg.num_layers; i++) { auto block = std::dynamic_pointer_cast(blocks["transformer_blocks." + std::to_string(i)]); - auto out = block->forward(ctx, - vx, - ax, - v_context, - a_context, - nullptr, - v_timestep_mod, - a_timestep_mod, - v_pe, - a_pe, - v_cross_pe, - a_cross_pe, - av_ca_video_scale_shift_timestep, - av_ca_audio_scale_shift_timestep, - av_ca_a2v_gate_noise_timestep, - av_ca_v2a_gate_noise_timestep, - v_prompt_timestep_mod, - a_prompt_timestep_mod); - vx = out.first; - ax = out.second; + const bool skip_video_self_attention = + skip_video_self_attention_blocks != nullptr && + std::find(skip_video_self_attention_blocks->begin(), + skip_video_self_attention_blocks->end(), + i) != skip_video_self_attention_blocks->end(); + auto out = block->forward(ctx, + vx, + ax, + v_context, + a_context, + nullptr, + v_timestep_mod, + a_timestep_mod, + v_pe, + a_pe, + v_cross_pe, + a_cross_pe, + av_ca_video_scale_shift_timestep, + av_ca_audio_scale_shift_timestep, + av_ca_a2v_gate_noise_timestep, + av_ca_v2a_gate_noise_timestep, + v_prompt_timestep_mod, + a_prompt_timestep_mod, + nullptr, + skip_video_self_attention); + vx = out.first; + ax = out.second; sd::ggml_graph_cut::mark_graph_cut(vx, "ltxav.transformer_blocks." + std::to_string(i), "vx"); sd::ggml_graph_cut::mark_graph_cut(ax, "ltxav.transformer_blocks." + std::to_string(i), "ax"); } @@ -1744,7 +1755,8 @@ namespace LTXV { const sd::Tensor& audio_timesteps_tensor = {}, int audio_length = 0, float frame_rate = 24.f, - const sd::Tensor& video_positions_tensor = {}) { + const sd::Tensor& video_positions_tensor = {}, + const std::vector* skip_video_self_attention_blocks = nullptr) { auto split_inputs = split_av_latents(x_tensor, audio_length); vx_input_cache = split_inputs.first; if (!audio_x_tensor.empty()) { @@ -1894,7 +1906,8 @@ namespace LTXV { video_cross_pe, audio_cross_pe, video_connector_pe, - audio_connector_pe); + audio_connector_pe, + skip_video_self_attention_blocks); auto out = merge_av_latents(compute_ctx, out_pair.first, out_pair.second); ggml_build_forward_expand(gf, out); return gf; @@ -1908,9 +1921,18 @@ namespace LTXV { const sd::Tensor& audio_timesteps = {}, int audio_length = 0, float frame_rate = 24.f, - const sd::Tensor& video_positions = {}) { + const sd::Tensor& video_positions = {}, + const std::vector* skip_video_self_attention_blocks = nullptr) { auto get_graph = [&]() -> ggml_cgraph* { - return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions); + return build_graph(x, + timesteps, + context, + audio_x, + audio_timesteps, + audio_length, + frame_rate, + video_positions, + skip_video_self_attention_blocks); }; auto out = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); return out; @@ -1929,7 +1951,8 @@ namespace LTXV { tensor_or_empty(extra->audio_timesteps), extra->audio_length, extra->frame_rate, - tensor_or_empty(extra->video_positions)); + tensor_or_empty(extra->video_positions), + extra->skip_video_self_attention_blocks); } void test(const std::string& x_path, diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 2b2f4dc78..f3d12ac7a 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2207,7 +2207,8 @@ class StableDiffusionGGML { audio_timesteps_tensor.empty() ? nullptr : &audio_timesteps_tensor, audio_length, frame_rate, - video_positions.empty() ? nullptr : &video_positions}; + video_positions.empty() ? nullptr : &video_positions, + local_skip_layers}; } else { diffusion_params.extra = std::monostate{}; } @@ -3655,6 +3656,7 @@ struct ImageGenerationLatents { int64_t ref_image_num = 0; int64_t video_conditioning_frame_count = 0; int64_t video_target_frame_count = 0; + bool video_conditioning_first = false; int audio_length = 0; }; @@ -3693,7 +3695,8 @@ static sd::Tensor build_ltxv_video_positions(int64_t width, int fps, int spatial_scale, int temporal_scale, - bool causal_temporal_positioning) { + bool causal_temporal_positioning, + bool keyframe_first = false) { GGML_ASSERT(width > 0 && height > 0 && target_latent_frames > 0); GGML_ASSERT(keyframe_latent_frames > 0); GGML_ASSERT(fps > 0); @@ -3702,41 +3705,38 @@ static sd::Tensor build_ltxv_video_positions(int64_t width, sd::Tensor positions({2, 3, total_tokens, 1}); int64_t token = 0; - for (int64_t t = 0; t < target_latent_frames; t++) { - float t_start = ltxv_latent_corner_to_pixel_frame(t, temporal_scale, causal_temporal_positioning) / static_cast(fps); - float t_end = ltxv_latent_corner_to_pixel_frame(t + 1, temporal_scale, causal_temporal_positioning) / static_cast(fps); - for (int64_t h = 0; h < height; h++) { - float h_start = static_cast(h * spatial_scale); - float h_end = static_cast((h + 1) * spatial_scale); - for (int64_t w = 0; w < width; w++) { - float w_start = static_cast(w * spatial_scale); - float w_end = static_cast((w + 1) * spatial_scale); - set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end); + auto append_positions = [&](bool keyframe) { + const int64_t frame_count = keyframe ? keyframe_latent_frames : target_latent_frames; + for (int64_t t = 0; t < frame_count; t++) { + const int64_t start_corner = (keyframe ? keyframe_frame_idx : 0) + t; + const int64_t end_corner = start_corner + 1; + float t_start = ltxv_latent_corner_to_pixel_frame( + start_corner, temporal_scale, causal_temporal_positioning); + float t_end = ltxv_latent_corner_to_pixel_frame( + end_corner, temporal_scale, causal_temporal_positioning); + if (keyframe && keyframe_pixel_frames == 1) { + t_end = t_start + 1.f; + } + t_start /= static_cast(fps); + t_end /= static_cast(fps); + for (int64_t h = 0; h < height; h++) { + float h_start = static_cast(h * spatial_scale); + float h_end = static_cast((h + 1) * spatial_scale); + for (int64_t w = 0; w < width; w++) { + float w_start = static_cast(w * spatial_scale); + float w_end = static_cast((w + 1) * spatial_scale); + set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end); + } } } - } + }; - for (int64_t t = 0; t < keyframe_latent_frames; t++) { - const int64_t start_corner = keyframe_frame_idx + t; - const int64_t end_corner = keyframe_frame_idx + t + 1; - float t_start = ltxv_latent_corner_to_pixel_frame( - start_corner, temporal_scale, causal_temporal_positioning); - float t_end = ltxv_latent_corner_to_pixel_frame( - end_corner, temporal_scale, causal_temporal_positioning); - if (keyframe_pixel_frames == 1) { - t_end = t_start + 1.f; - } - t_start /= static_cast(fps); - t_end /= static_cast(fps); - for (int64_t h = 0; h < height; h++) { - float h_start = static_cast(h * spatial_scale); - float h_end = static_cast((h + 1) * spatial_scale); - for (int64_t w = 0; w < width; w++) { - float w_start = static_cast(w * spatial_scale); - float w_end = static_cast((w + 1) * spatial_scale); - set_ltxv_video_position(&positions, token++, t_start, t_end, h_start, h_end, w_start, w_end); - } - } + if (keyframe_first) { + append_positions(true); + append_positions(false); + } else { + append_positions(false); + append_positions(true); } return positions; @@ -4931,7 +4931,8 @@ static std::optional prepare_video_generation_latents(sd auto apply_video_condition_by_keyframe_index = [&](const sd::Tensor& keyframes, int frame_idx, - const char* name) -> bool { + const char* name, + bool condition_first = false) -> bool { int64_t keyframe_frames = keyframes.shape()[2]; if (keyframe_frames <= 0 || keyframes.shape()[0] != latents.init_latent.shape()[0] || keyframes.shape()[1] != latents.init_latent.shape()[1] || @@ -4942,15 +4943,21 @@ static std::optional prepare_video_generation_latents(sd latents.video_target_frame_count = latents.init_latent.shape()[2]; latents.video_conditioning_frame_count = keyframe_frames; - latents.init_latent = sd::ops::concat(latents.init_latent, keyframes, 2); + latents.video_conditioning_first = condition_first; - auto keyframe_mask = sd::full({keyframes.shape()[0], - keyframes.shape()[1], - keyframes.shape()[2], - 1, - 1}, + auto keyframe_mask = sd::full({keyframes.shape()[0], + keyframes.shape()[1], + keyframes.shape()[2], + 1, + 1}, conditioned_mask); - latents.denoise_mask = sd::ops::concat(latents.denoise_mask, keyframe_mask, 2); + if (condition_first) { + latents.init_latent = sd::ops::concat(keyframes, latents.init_latent, 2); + latents.denoise_mask = sd::ops::concat(keyframe_mask, latents.denoise_mask, 2); + } else { + latents.init_latent = sd::ops::concat(latents.init_latent, keyframes, 2); + latents.denoise_mask = sd::ops::concat(latents.denoise_mask, keyframe_mask, 2); + } latents.video_positions = build_ltxv_video_positions(latents.init_latent.shape()[0], latents.init_latent.shape()[1], latents.video_target_frame_count, @@ -4960,7 +4967,8 @@ static std::optional prepare_video_generation_latents(sd request->fps, request->vae_scale_factor, 8, - true); + true, + condition_first); return true; }; @@ -5011,7 +5019,8 @@ static std::optional prepare_video_generation_latents(sd } if (!apply_video_condition_by_keyframe_index(reference_latents, 0, - "IC-LoRA reference")) { + "IC-LoRA reference", + true)) { return std::nullopt; } const float reference_strength = @@ -5019,8 +5028,8 @@ static std::optional prepare_video_generation_latents(sd const int64_t reference_frames = reference_latents.shape()[2]; sd::ops::fill_slice(&latents.denoise_mask, 2, - latents.denoise_mask.shape()[2] - reference_frames, - latents.denoise_mask.shape()[2], + 0, + reference_frames, 1.f - reference_strength); LOG_INFO("LTX IC-LoRA conditioned on %zu static reference image(s), strength %.3f", reference_images.size(), @@ -5828,7 +5837,8 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, if (latents.video_conditioning_frame_count > 0) { int64_t target_frames = latents.video_target_frame_count > 0 ? latents.video_target_frame_count : final_latent.shape()[2] - latents.video_conditioning_frame_count; - final_latent = sd::ops::slice(final_latent, 2, 0, target_frames); + int64_t target_start = latents.video_conditioning_first ? latents.video_conditioning_frame_count : 0; + final_latent = sd::ops::slice(final_latent, 2, target_start, target_start + target_frames); } if (latents.ref_image_num > 0) {