Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions src/boring_semantic_layer/calc_compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,30 @@ def __getitem__(self, name: str):
return self._virtual_agg_tbl[resolved]
return self._base_tbl[name]

def _rebind_to_totals(self, x: Any):
"""Re-point every measure reference inside *x* at the totals table.

Returns ``None`` when *x* contains no measure references, in which
case the caller falls back to the reduction/windowed handling —
``t.all(t.raw_column)`` has no measure formula to re-apply.
"""
try:
op = _to_op(x)
vt_op = _to_op(self._virtual_agg_tbl)
totals_op = _to_op(self._totals_virtual_agg_tbl)
referenced = {
node.name
for node in _walk(op)
if isinstance(node, Field) and id(node.rel) == id(vt_op)
}
if not referenced:
return None
subs = {Field(vt_op, name): Field(totals_op, name) for name in referenced}
return op.replace(subs).to_expr()
except Exception as exc:
logger.debug("IbisCalcScope.all() totals rebind swallowed: %s", exc)
return None

def all(self, x: Any):
"""Resolve a measure reference to its totals-table column.

Expand Down Expand Up @@ -329,6 +353,16 @@ def all(self, x: Any):
if isinstance(op, Field) and id(op.rel) == id(_to_op(self._virtual_agg_tbl)):
return self._totals_virtual_agg_tbl[op.name]

# An expression *built from* measure references is the same
# request, one level up: evaluate it against the totals table
# rather than the per-group one. Falling through to the
# windowed-sum shape below made ``t.all(m)`` and
# ``t.all(m * 1)`` mean different things — for a mean measure
# the second silently summed the per-group means.
over_totals = self._rebind_to_totals(x)
if over_totals is not None:
return over_totals

Reduction = getattr(ibis_ops, "Reduction", None)
if Reduction is not None:
if isinstance(op, Reduction):
Expand Down
15 changes: 13 additions & 2 deletions src/boring_semantic_layer/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,12 +263,23 @@ def mutate(self, **post):
chain order and materialized, preserving the historical
``.mutate()`` semantics without a dedicated operator node.
"""
from .ops import SemanticJoinOp, SemanticTableOp, _has_prior_aggregate, _resolve_expr
from .ops import (
SemanticJoinOp,
SemanticTableOp,
_has_prior_aggregate,
_non_additive_result_columns,
_resolve_expr,
)

if _has_prior_aggregate(self.op()):
tbl = self.op().to_untagged()
# Only the aggregated rows are available here, so t.all() can only
# window-sum them; pass which columns that would misrepresent.
non_additive = _non_additive_result_columns(self.op())
for name, fn in post.items():
proxy = MeasureScope(_tbl=tbl, _known=[], _post_agg=True)
proxy = MeasureScope(
_tbl=tbl, _known=[], _post_agg=True, _non_additive=non_additive
)
resolved = _resolve_expr(fn, proxy)
tbl = tbl.mutate(resolved.name(name))
return _build_post_aggregate_model(self.op(), tbl)
Expand Down
68 changes: 64 additions & 4 deletions src/boring_semantic_layer/measure_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,15 @@ class MeasureScope:
converter=tuple,
alias="_prefer_known",
)
#: Result columns whose window sum is *not* their overall value (means,
#: medians, distinct counts, calc measures). ``t.all()`` refuses these
#: rather than returning a sum with no meaning. Empty means "not
#: classified" — the historical behaviour applies.
non_additive: frozenset[str] = field(
factory=frozenset,
converter=frozenset,
alias="_non_additive",
)

def __attrs_post_init__(self):
object.__setattr__(self, "known_set", frozenset(self.known))
Expand Down Expand Up @@ -166,17 +175,68 @@ def all(self, ref):
from ._xorq import ibis as ibis_mod

if isinstance(ref, str):
return self.tbl[ref].sum().over(ibis_mod.window())
self._reject_non_additive(ref)
return _float_total(self.tbl[ref].sum().over(ibis_mod.window()))

if hasattr(ref, "__class__") and "ibis" in str(type(ref).__module__):
if "Scalar" in type(ref).__name__:
return ref.over(ibis_mod.window())
return ref.sum().over(ibis_mod.window())
self._reject_non_additive(_column_name_of(ref))
return _float_total(ref.sum().over(ibis_mod.window()))

raise TypeError(
"t.all(...) expects a string column name or an ibis expression",
)

def _reject_non_additive(self, name: str | None) -> None:
"""Refuse a total this scope cannot compute correctly.

Post-aggregation, the only rows available are the grouped ones, so
the total can only be a window sum over them. For a mean, median,
distinct count or ratio that sum is not the overall value, and
returning it silently produced answers that disagreed with the same
formula written as a calc measure.
"""
if name is None or name not in self.non_additive:
return
raise NonAdditiveTotalError(
f"t.all({name!r}) cannot be computed after aggregation: {name!r} is "
"not additive, so summing its per-group values is not its overall "
"value. Define the derivation on the model instead, where the total "
"is computed from the underlying rows:\n"
f" model.with_measures(share=lambda t: t.{name} / t.all(t.{name}))\n"
"or place the .mutate() directly on the aggregate (before "
"filter/order_by/limit), which routes through the same path."
)


class NonAdditiveTotalError(ValueError):
"""``t.all()`` was asked for a total that post-aggregation rows can't give."""


def _column_name_of(ref) -> str | None:
"""Best-effort column name for an ibis value expression."""
try:
name = ref.get_name()
except Exception:
return None
return name if isinstance(name, str) else None


def _float_total(total):
"""Cast an integral total to float so ``measure / total`` is a true ratio.

The calc-measure path gives its virtual columns a float64 schema for the
same reason: with two integer operands some engines (xorq's DataFusion
among them) do integer division, and ``30 / 160`` came back as ``0``.
"""
try:
if total.type().is_integer():
return total.cast("float64")
except Exception:
pass
return total


@frozen(kw_only=True, slots=True)
class ColumnScope:
Expand Down Expand Up @@ -206,12 +266,12 @@ def all(self, ref):
from ._xorq import ibis as ibis_mod

if isinstance(ref, str):
return self.tbl[ref].sum().over(ibis_mod.window())
return _float_total(self.tbl[ref].sum().over(ibis_mod.window()))

if hasattr(ref, "__class__") and "ibis" in str(type(ref).__module__):
if "Scalar" in type(ref).__name__:
return ref.over(ibis_mod.window())
return ref.sum().over(ibis_mod.window())
return _float_total(ref.sum().over(ibis_mod.window()))

raise TypeError(
"t.all(...) expects a string column name or an ibis expression",
Expand Down
Loading
Loading