diff --git a/src/matcher.rs b/src/matcher.rs index 5adca82..abc9984 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -358,6 +358,7 @@ fn process_atomic( col: usize, input_text: &str, item_index: usize, + span: (usize, usize), sem: &mut SemanticsCore, ) -> Result<(), SemErr> { if atomic.idd == Idd::Skip { @@ -376,6 +377,7 @@ fn process_atomic( row, col, ty, + span, }); for action_spec in &atomic.actions { let action = instantiate_action(ItemId::Cell(item_id), action_spec, sem)?; @@ -384,6 +386,24 @@ fn process_atomic( Ok(()) } +/// Non-empty `java_trim`med parts of a literal split as +/// (original part position, trimmed part, byte span in the original cell +/// text); `base` is the offset of `text` within that cell text. +fn split_with_spans(delim: &str, text: &str, base: usize) -> Vec<(usize, String, (usize, usize))> { + let mut out = Vec::new(); + let mut start = 0usize; + for (i, part) in split_literal(delim, text).into_iter().enumerate() { + let trimmed = java_trim(&part); + if !trimmed.is_empty() { + let lead = part.len() - part.trim_start_matches(|c: char| c <= ' ').len(); + let from = base + start + lead; + out.push((i, trimmed.to_string(), (from, from + trimmed.len()))); + } + start += part.len() + delim.len(); + } + out +} + fn process_delimited( d: &DelimitedSpec, row: usize, @@ -391,12 +411,8 @@ fn process_delimited( text: &str, sem: &mut SemanticsCore, ) -> Result<(), SemErr> { - let parts = split_literal(&d.delimiter, text); - for (i, part) in parts.iter().enumerate() { - let part = java_trim(part); - if !part.is_empty() { - process_atomic(&d.atom, row, col, part, i, sem)?; - } + for (i, part, span) in split_with_spans(&d.delimiter, text, 0) { + process_atomic(&d.atom, row, col, &part, i, span, sem)?; } Ok(()) } @@ -444,16 +460,13 @@ fn process_compound( let substring = &text[pos..end_pos]; match &seg.spec { ContentSpec::Atomic(a) => { - process_atomic(a, row, col, substring, item_index, sem)?; + process_atomic(a, row, col, substring, item_index, (pos, end_pos), sem)?; item_index += 1; } ContentSpec::Delimited(d) => { - for part in split_literal(&d.delimiter, substring) { - let trimmed = java_trim(&part); - if !trimmed.is_empty() { - process_atomic(&d.atom, row, col, trimmed, item_index, sem)?; - item_index += 1; - } + for (_, part, span) in split_with_spans(&d.delimiter, substring, pos) { + process_atomic(&d.atom, row, col, &part, item_index, span, sem)?; + item_index += 1; } } _ => unreachable!("compound segment restricted to atomic/delimited"), @@ -476,7 +489,8 @@ fn process_content_spec( match cs { ContentSpec::Atomic(a) => { let text = syntax.cell(row, col).text.clone(); - process_atomic(a, row, col, &text, 0, sem) + let span = (0, text.len()); + process_atomic(a, row, col, &text, 0, span, sem) } ContentSpec::Delimited(d) => { let text = syntax.cell(row, col).text.clone(); diff --git a/src/py.rs b/src/py.rs index 6480f8e..fb49883 100644 --- a/src/py.rs +++ b/src/py.rs @@ -85,6 +85,7 @@ pub fn call_item_filter( ty: it.ty, row: it.row, col: it.col, + span: it.span, table: table.as_ref().map(|t| t.clone_ref(py)), }; let res = func @@ -635,6 +636,7 @@ pub struct PyCellDerivedItem { pub ty: sp::ItemType, pub row: usize, pub col: usize, + pub span: (usize, usize), pub table: Option>, } @@ -644,6 +646,12 @@ impl PyCellDerivedItem { fn str(&self) -> String { self.s.clone() } + /// Byte range of the item's source segment within the raw cell text + /// (before extractors). + #[getter] + fn span(&self) -> (usize, usize) { + self.span + } #[getter] fn tags(&self) -> Vec { self.tags.clone() @@ -2610,6 +2618,7 @@ impl PyTableSemantics { ty: it.ty, row: it.row, col: it.col, + span: it.span, table: Some(self.table.clone_ref(py)), }) .collect() diff --git a/src/semantics.rs b/src/semantics.rs index 23c1773..4d02ef5 100644 --- a/src/semantics.rs +++ b/src/semantics.rs @@ -21,6 +21,10 @@ pub struct CellItem { pub row: usize, pub col: usize, pub ty: ItemType, + /// Byte range of the item's source segment within the raw cell text + /// (before extractors): atomic content spans the whole text, delimited + /// and compound content spans the segment the item was derived from. + pub span: (usize, usize), } /// Context-derived item. diff --git a/src/tests.rs b/src/tests.rs index 06111f8..e4d18dc 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -112,6 +112,25 @@ fn end_to_end_match_and_interpret() { let sem = match_atp(&pattern, &mut syntax, Vec::new()) .unwrap() .expect("pattern must match"); + + // Item spans: the compound cell "0 Jan" (row 1, col 1) yields two VAL + // items whose spans point at the segments of the raw cell text. + let compound: Vec<_> = sem + .cell_items + .iter() + .filter(|it| it.row == 1 && it.col == 1) + .collect(); + assert_eq!(compound.len(), 2); + assert_eq!((compound[0].s.as_str(), compound[0].span), ("0", (0, 1))); + assert_eq!((compound[1].s.as_str(), compound[1].span), ("Jan", (2, 5))); + // Atomic cells span their whole text. + let atomic = sem + .cell_items + .iter() + .find(|it| it.row == 1 && it.col == 0) + .unwrap(); + assert_eq!(atomic.span, (0, "IKT".len())); + let rs = interpret(&InterpreterCfg::default(), &syntax, &sem, None).unwrap(); assert_eq!(rs.schema.attributes, vec!["ND", "AIRLINE", "AIRPORT", "MON"]);