From 2926cf8b32dbcf923bc99f4f28dccbc8f74d95ad Mon Sep 17 00:00:00 2001 From: Liam Abourousse <75449798+Bardakor@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:49:34 +0300 Subject: [PATCH 1/3] perf(layout): defer masked chunk evaluation Signed-off-by: Liam Abourousse <75449798+Bardakor@users.noreply.github.com> --- vortex-layout/src/layouts/chunked/reader.rs | 73 ++++++++++++++++++--- 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/vortex-layout/src/layouts/chunked/reader.rs b/vortex-layout/src/layouts/chunked/reader.rs index b5f4f5b438d..f741a5b666e 100644 --- a/vortex-layout/src/layouts/chunked/reader.rs +++ b/vortex-layout/src/layouts/chunked/reader.rs @@ -41,7 +41,7 @@ use crate::segments::SegmentSource; pub struct ChunkedReader { layout: ChunkedLayout, name: Arc, - lazy_children: LazyReaderChildren, + lazy_children: Arc, /// Lazily computed classification of which chunks register no interior splits, letting /// [`ChunkedReader::register_splits`] avoid materializing those chunks' readers. chunk_skips: OnceCell, @@ -98,7 +98,7 @@ impl ChunkedReader { Self { layout, name, - lazy_children, + lazy_children: Arc::new(lazy_children), chunk_skips: OnceCell::new(), } } @@ -305,9 +305,17 @@ impl LayoutReader for ChunkedReader { let mut chunk_evals = vec![]; for (chunk_idx, _, chunk_range, mask_range) in self.ranges(row_range) { + let chunk_mask = mask.slice(mask_range); + if chunk_mask.all_false() { + // The input mask already excludes this chunk. Returning all true means the child + // contributes no additional pruning while avoiding reader construction entirely. + chunk_evals.push(MaskFuture::new_true(chunk_mask.len())); + continue; + } + let chunk_reader = self.chunk_reader(chunk_idx)?; let chunk_eval = chunk_reader - .pruning_evaluation(&chunk_range, expr, mask.slice(mask_range)) + .pruning_evaluation(&chunk_range, expr, chunk_mask) .map_err(|err| { err.with_context(format!( "While evaluating pruning filter on chunk {chunk_idx}" @@ -351,12 +359,29 @@ impl LayoutReader for ChunkedReader { let mut chunk_evals = vec![]; for (chunk_idx, _, chunk_range, mask_range) in self.ranges(row_range) { - let chunk_reader = self.chunk_reader(chunk_idx)?; - let chunk_eval = chunk_reader - .filter_evaluation(&chunk_range, expr, mask.slice(mask_range)) - .map_err(|err| { - err.with_context(format!("While evaluating filter on chunk {chunk_idx}")) - })?; + let lazy_children = Arc::clone(&self.lazy_children); + let expr = expr.clone(); + let chunk_mask = mask.slice(mask_range); + let chunk_len = chunk_mask.len(); + + let chunk_eval = MaskFuture::new(chunk_len, async move { + let chunk_mask = chunk_mask.await?; + if chunk_mask.all_false() { + return Ok(chunk_mask); + } + + let chunk_reader = Arc::clone(lazy_children.get(chunk_idx)?); + let chunk_eval = chunk_reader + .filter_evaluation( + &chunk_range, + &expr, + MaskFuture::ready(chunk_mask), + ) + .map_err(|err| { + err.with_context(format!("While evaluating filter on chunk {chunk_idx}")) + })?; + chunk_eval.await + }); chunk_evals.push(chunk_eval); } @@ -442,6 +467,7 @@ mod test { use vortex_buffer::buffer; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; + use vortex_mask::Mask; use vortex_session::registry::ReadContext; use crate::LayoutRef; @@ -546,6 +572,35 @@ mod test { assert_eq!(splits, expected.into_iter().collect::>()); } + #[test] + fn test_filter_skips_fully_masked_chunks() { + let layout = nested_chunked_layout(); + block_on(|_handle| async { + let reader = layout + .new_reader( + "".into(), + Arc::new(TestSegments::default()), + &SESSION, + &Default::default(), + ) + .unwrap(); + let expr = root().bind(reader.dtype()).unwrap(); + let row_count = usize::try_from(layout.row_count()).unwrap(); + + let result = reader + .filter_evaluation( + &(0..layout.row_count()), + &expr, + MaskFuture::ready(Mask::new_false(row_count)), + ) + .unwrap() + .await + .unwrap(); + + assert!(result.all_false()); + }) + } + #[rstest] fn test_chunked_evaluator( #[from(chunked_layout)] (segments, layout): (Arc, LayoutRef), From b1b8bc1b0411e5d70ef45c58ef444d637247b67b Mon Sep 17 00:00:00 2001 From: Liam Abourousse <75449798+Bardakor@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:50:44 +0300 Subject: [PATCH 2/3] perf(layout): prune before materializing data children Signed-off-by: Liam Abourousse <75449798+Bardakor@users.noreply.github.com> --- vortex-layout/src/layouts/zoned/reader.rs | 24 ++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/vortex-layout/src/layouts/zoned/reader.rs b/vortex-layout/src/layouts/zoned/reader.rs index 52a57c1b92b..162898aa382 100644 --- a/vortex-layout/src/layouts/zoned/reader.rs +++ b/vortex-layout/src/layouts/zoned/reader.rs @@ -140,13 +140,18 @@ impl LayoutReader for ZonedReader { mask: Mask, ) -> VortexResult { trace!("Stats pruning evaluation: {} - {}", &self.name, expr); - let data_eval = self - .data_child()? - .pruning_evaluation(row_range, expr, mask.clone())?; let Some(pruning_mask_future) = self.pruning.pruning_mask_future(expr.clone()) else { trace!("Stats pruning evaluation: not prune-able {expr}"); - return Ok(data_eval); + let lazy_children = Arc::clone(&self.lazy_children); + let row_range = row_range.clone(); + let expr = expr.clone(); + return Ok(MaskFuture::new(mask.len(), async move { + let data_child = Arc::clone(lazy_children.get(0)?); + data_child + .pruning_evaluation(&row_range, &expr, mask)? + .await + })); }; let row_count = row_range.end - row_range.start; @@ -169,6 +174,8 @@ impl LayoutReader for ZonedReader { .try_collect()?; let name = Arc::clone(&self.name); + let lazy_children = Arc::clone(&self.lazy_children); + let row_range = row_range.clone(); let expr = expr.clone(); Ok(MaskFuture::new(mask.len(), async move { @@ -188,8 +195,15 @@ impl LayoutReader for ZonedReader { let mask_density = mask.density(); let mut stats_mask = mask.bitand(&stats_mask); - // Forward to data child for further pruning. + // Only materialize the data child after statistics have pruned the input, and pass + // that sparse mask down so chunked readers can skip fully excluded chunks. if !stats_mask.all_false() { + let data_child = Arc::clone(lazy_children.get(0)?); + let data_eval = data_child.pruning_evaluation( + &row_range, + &expr, + stats_mask.clone(), + )?; let data_mask = data_eval.await?; stats_mask = stats_mask.bitand(&data_mask); } From 9a2cebb6cb5c67b4c49866e5756b55100cd6fefa Mon Sep 17 00:00:00 2001 From: Liam Abourousse <75449798+Bardakor@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:52:39 +0300 Subject: [PATCH 3/3] fix(layout): preserve excluded pruning rows Signed-off-by: Liam Abourousse <75449798+Bardakor@users.noreply.github.com> --- vortex-layout/src/layouts/chunked/reader.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vortex-layout/src/layouts/chunked/reader.rs b/vortex-layout/src/layouts/chunked/reader.rs index f741a5b666e..7eed0e1ca24 100644 --- a/vortex-layout/src/layouts/chunked/reader.rs +++ b/vortex-layout/src/layouts/chunked/reader.rs @@ -307,9 +307,8 @@ impl LayoutReader for ChunkedReader { for (chunk_idx, _, chunk_range, mask_range) in self.ranges(row_range) { let chunk_mask = mask.slice(mask_range); if chunk_mask.all_false() { - // The input mask already excludes this chunk. Returning all true means the child - // contributes no additional pruning while avoiding reader construction entirely. - chunk_evals.push(MaskFuture::new_true(chunk_mask.len())); + // Preserve already-excluded rows without materializing the child reader. + chunk_evals.push(MaskFuture::ready(chunk_mask)); continue; }