From 1b1461859f56e30e5950213f74914397270b8619 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Wed, 17 Jun 2026 13:17:26 +0200 Subject: [PATCH] feat(core)!: remove deprecated Time, Timestamp, TimestampTZ types and literals Removes types, literals, and function mappings that were previously deprecated in substrait-java and are removed in Substrait v0.88.0: - Type.Time, Type.Timestamp, Type.TimestampTZ - Expression.TimeLiteral, Expression.TimestampLiteral, Expression.TimestampTZLiteral - TypeCreator.TIME, TIMESTAMP, TIMESTAMP_TZ - All visitor methods and proto converter cases for the above Also updates the substrait submodule to v0.88.0, which removes these types from the proto definitions and ANTLR grammar. Additional fixes required by v0.88.0: - ParseToPojo: handle new integer_parameter(x) function call in the type derivation grammar used by functions_datetime.yaml - ToTypeString: add missing visit(ParameterizedType.PrecisionTime) - EnumConverter: update extract function keys for precision_time (req_time -> req_pt) and remove stale req_ts/req_tstz_str entries - FunctionConversionTest: update test signatures to v0.88.0 function keys and add precision arg to strptime_time/strptime_timestamp BREAKING CHANGE: Type.Time, Type.Timestamp, Type.TimestampTZ and their corresponding expression literals and visitor methods are removed; use the precision variants (Type.PrecisionTime, Type.PrecisionTimestamp, Type.PrecisionTimestampTZ) instead. Part of https://github.com/substrait-io/substrait-java/issues/800 Closes https://github.com/substrait-io/substrait-java/issues/804 --- .../expression/AbstractExpressionVisitor.java | 39 ---- .../io/substrait/expression/Expression.java | 104 ----------- .../expression/ExpressionCreator.java | 102 ----------- .../expression/ExpressionVisitor.java | 30 --- .../proto/ExpressionProtoConverter.java | 18 -- .../proto/ProtoExpressionConverter.java | 6 - .../io/substrait/function/ToTypeString.java | 20 +- .../ExpressionCopyOnWriteVisitor.java | 18 -- .../substrait/relation/VirtualTableScan.java | 15 -- .../io/substrait/type/StringTypeVisitor.java | 15 -- .../src/main/java/io/substrait/type/Type.java | 67 ------- .../java/io/substrait/type/TypeCreator.java | 12 -- .../java/io/substrait/type/TypeVisitor.java | 48 ----- .../io/substrait/type/parser/ParseToPojo.java | 31 ++-- .../type/proto/BaseProtoConverter.java | 15 -- .../substrait/type/proto/BaseProtoTypes.java | 6 - .../type/proto/ProtoTypeConverter.java | 6 - .../type/proto/TypeProtoConverter.java | 6 - .../substrait/function/ToTypeStringTest.java | 3 - .../proto/DynamicParameterRoundtripTest.java | 4 +- .../type/proto/SortRelRoundtripTest.java | 2 +- .../type/proto/TestTypeRoundtrip.java | 10 +- .../examples/util/ExpressionStringify.java | 20 -- .../examples/util/TypeStringify.java | 20 -- .../isthmus/SimpleExtensionToSqlOperator.java | 15 -- .../io/substrait/isthmus/TypeConverter.java | 15 -- .../isthmus/expression/EnumConverter.java | 11 +- .../expression/ExpressionRexConverter.java | 23 --- .../IgnoreNullableAndParameters.java | 33 ---- .../isthmus/FunctionConversionTest.java | 171 ++++++------------ spark/spark_dialect.yaml | 16 +- .../IgnoreNullableAndParameters.scala | 9 - substrait | 2 +- 33 files changed, 84 insertions(+), 828 deletions(-) diff --git a/core/src/main/java/io/substrait/expression/AbstractExpressionVisitor.java b/core/src/main/java/io/substrait/expression/AbstractExpressionVisitor.java index 3914587f3..79e4392d2 100644 --- a/core/src/main/java/io/substrait/expression/AbstractExpressionVisitor.java +++ b/core/src/main/java/io/substrait/expression/AbstractExpressionVisitor.java @@ -152,19 +152,6 @@ public O visit(Expression.BinaryLiteral expr, C context) throws E { return visitFallback(expr, context); } - /** - * Visits a time literal. - * - * @param expr the time literal - * @param context the visitation context - * @return the visit result - * @throws E if visitation fails - */ - @Override - public O visit(Expression.TimeLiteral expr, C context) throws E { - return visitFallback(expr, context); - } - /** * Visits a precision time literal. * @@ -191,32 +178,6 @@ public O visit(Expression.DateLiteral expr, C context) throws E { return visitFallback(expr, context); } - /** - * Visits a timestamp literal. - * - * @param expr the timestamp literal - * @param context the visitation context - * @return the visit result - * @throws E if visitation fails - */ - @Override - public O visit(Expression.TimestampLiteral expr, C context) throws E { - return visitFallback(expr, context); - } - - /** - * Visits a timestamp-with-time-zone literal. - * - * @param expr the timestamp TZ literal - * @param context the visitation context - * @return the visit result - * @throws E if visitation fails - */ - @Override - public O visit(Expression.TimestampTZLiteral expr, C context) throws E { - return visitFallback(expr, context); - } - /** * Visits a precision timestamp literal. * diff --git a/core/src/main/java/io/substrait/expression/Expression.java b/core/src/main/java/io/substrait/expression/Expression.java index 5641a3758..db17fed9c 100644 --- a/core/src/main/java/io/substrait/expression/Expression.java +++ b/core/src/main/java/io/substrait/expression/Expression.java @@ -421,75 +421,6 @@ public R accept( } } - /** - * @deprecated {@link TimestampLiteral} is deprecated in favor of {@link - * PrecisionTimestampLiteral} - */ - @Value.Immutable - @Deprecated - abstract class TimestampLiteral implements Literal { - /** - * Returns the timestamp value of this literal in microseconds since epoch. - * - * @return the timestamp value - */ - public abstract long value(); - - @Override - public Type getType() { - return Type.withNullability(nullable()).TIMESTAMP; - } - - /** - * Creates a new builder for constructing a TimestampLiteral. - * - * @return a new builder instance - */ - public static ImmutableExpression.TimestampLiteral.Builder builder() { - return ImmutableExpression.TimestampLiteral.builder(); - } - - @Override - public R accept( - ExpressionVisitor visitor, C context) throws E { - return visitor.visit(this, context); - } - } - - /** - * @deprecated {@link TimeLiteral} is deprecated in favor of {@link PrecisionTimeLiteral} - */ - @Value.Immutable - @Deprecated - abstract class TimeLiteral implements Literal { - /** - * Returns the time value of this literal in microseconds since midnight. - * - * @return the time value - */ - public abstract long value(); - - @Override - public Type getType() { - return Type.withNullability(nullable()).TIME; - } - - /** - * Creates a new builder for constructing a TimeLiteral. - * - * @return a new builder instance - */ - public static ImmutableExpression.TimeLiteral.Builder builder() { - return ImmutableExpression.TimeLiteral.builder(); - } - - @Override - public R accept( - ExpressionVisitor visitor, C context) throws E { - return visitor.visit(this, context); - } - } - /** Represents a time literal with configurable precision. */ @Value.Immutable abstract class PrecisionTimeLiteral implements Literal { @@ -559,41 +490,6 @@ public R accept( } } - /** - * @deprecated {@link TimestampTZLiteral} is deprecated in favor of {@link - * PrecisionTimestampTZLiteral} - */ - @Value.Immutable - @Deprecated - abstract class TimestampTZLiteral implements Literal { - /** - * Returns the timestamp with timezone value of this literal in microseconds since epoch. - * - * @return the timestamp value - */ - public abstract long value(); - - @Override - public Type getType() { - return Type.withNullability(nullable()).TIMESTAMP_TZ; - } - - /** - * Creates a new builder for constructing a TimestampTZLiteral. - * - * @return a new builder instance - */ - public static ImmutableExpression.TimestampTZLiteral.Builder builder() { - return ImmutableExpression.TimestampTZLiteral.builder(); - } - - @Override - public R accept( - ExpressionVisitor visitor, C context) throws E { - return visitor.visit(this, context); - } - } - /** Represents a timestamp literal with configurable precision. */ @Value.Immutable abstract class PrecisionTimestampLiteral implements Literal { diff --git a/core/src/main/java/io/substrait/expression/ExpressionCreator.java b/core/src/main/java/io/substrait/expression/ExpressionCreator.java index f7fe5fd63..b6e75e677 100644 --- a/core/src/main/java/io/substrait/expression/ExpressionCreator.java +++ b/core/src/main/java/io/substrait/expression/ExpressionCreator.java @@ -8,7 +8,6 @@ import io.substrait.util.DecimalUtil; import java.math.BigDecimal; import java.nio.ByteBuffer; -import java.time.Instant; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; @@ -162,19 +161,6 @@ public static Expression.DateLiteral date(boolean nullable, int value) { return Expression.DateLiteral.builder().nullable(nullable).value(value).build(); } - /** - * Creates a time literal expression. - * - * @param nullable whether the literal can be null - * @param value the time value in microseconds since midnight - * @return a TimeLiteral expression - * @deprecated Time is deprecated in favor of PrecisionTime - */ - @Deprecated - public static Expression.TimeLiteral time(boolean nullable, long value) { - return Expression.TimeLiteral.builder().nullable(nullable).value(value).build(); - } - /** * Creates a precision time literal expression. * @@ -209,94 +195,6 @@ public static Expression.PrecisionTimeLiteral precisionTime(boolean nullable, Lo return precisionTime(nullable, epochNano, 9); } - /** - * Creates a timestamp literal expression. - * - * @param nullable whether the literal can be null - * @param value the timestamp value in microseconds since epoch - * @return a TimestampLiteral expression - * @deprecated Timestamp is deprecated in favor of PrecisionTimestamp - */ - @Deprecated - public static Expression.TimestampLiteral timestamp(boolean nullable, long value) { - return Expression.TimestampLiteral.builder().nullable(nullable).value(value).build(); - } - - /** - * Creates a timestamp literal expression from a LocalDateTime. - * - * @param nullable whether the literal can be null - * @param value the LocalDateTime value (interpreted as UTC) - * @return a TimestampLiteral expression - * @deprecated Timestamp is deprecated in favor of PrecisionTimestamp - */ - @Deprecated - public static Expression.TimestampLiteral timestamp(boolean nullable, LocalDateTime value) { - long epochMicro = - TimeUnit.SECONDS.toMicros(value.toEpochSecond(ZoneOffset.UTC)) - + TimeUnit.NANOSECONDS.toMicros(value.getNano()); - return timestamp(nullable, epochMicro); - } - - /** - * Creates a timestamp literal expression from date/time components. - * - * @param nullable whether the literal can be null - * @param year the year - * @param month the month (1-12) - * @param dayOfMonth the day of month (1-31) - * @param hour the hour (0-23) - * @param minute the minute (0-59) - * @param second the second (0-59) - * @param micros the microseconds (0-999999) - * @return a TimestampLiteral expression - * @deprecated Timestamp is deprecated in favor of PrecisionTimestamp - */ - @Deprecated - public static Expression.TimestampLiteral timestamp( - boolean nullable, - int year, - int month, - int dayOfMonth, - int hour, - int minute, - int second, - int micros) { - return timestamp( - nullable, - LocalDateTime.of(year, month, dayOfMonth, hour, minute, second) - .withNano((int) TimeUnit.MICROSECONDS.toNanos(micros))); - } - - /** - * Creates a timestamp with timezone literal expression. - * - * @param nullable whether the literal can be null - * @param value the timestamp value in microseconds since epoch - * @return a TimestampTZLiteral expression - * @deprecated TimestampTZ is deprecated in favor of PrecisionTimestampTZ - */ - @Deprecated - public static Expression.TimestampTZLiteral timestampTZ(boolean nullable, long value) { - return Expression.TimestampTZLiteral.builder().nullable(nullable).value(value).build(); - } - - /** - * Creates a timestamp with timezone literal expression from an Instant. - * - * @param nullable whether the literal can be null - * @param value the Instant value - * @return a TimestampTZLiteral expression - * @deprecated TimestampTZ is deprecated in favor of PrecisionTimestampTZ - */ - @Deprecated - public static Expression.TimestampTZLiteral timestampTZ(boolean nullable, Instant value) { - long epochMicro = - TimeUnit.SECONDS.toMicros(value.getEpochSecond()) - + TimeUnit.NANOSECONDS.toMicros(value.getNano()); - return timestampTZ(nullable, epochMicro); - } - /** * Creates a precision timestamp literal expression. * diff --git a/core/src/main/java/io/substrait/expression/ExpressionVisitor.java b/core/src/main/java/io/substrait/expression/ExpressionVisitor.java index c0f97ee47..5f72f7fe9 100644 --- a/core/src/main/java/io/substrait/expression/ExpressionVisitor.java +++ b/core/src/main/java/io/substrait/expression/ExpressionVisitor.java @@ -111,16 +111,6 @@ public interface ExpressionVisitor bldr.setNullable(expr.nullable()).setBinary(expr.value())); } - @Override - public Expression visit( - io.substrait.expression.Expression.TimeLiteral expr, EmptyVisitationContext context) { - return lit(bldr -> bldr.setNullable(expr.nullable()).setTime(expr.value())); - } - @Override public Expression visit( io.substrait.expression.Expression.PrecisionTimeLiteral expr, @@ -203,18 +197,6 @@ public Expression visit( return lit(bldr -> bldr.setNullable(expr.nullable()).setDate(expr.value())); } - @Override - public Expression visit( - io.substrait.expression.Expression.TimestampLiteral expr, EmptyVisitationContext context) { - return lit(bldr -> bldr.setNullable(expr.nullable()).setTimestamp(expr.value())); - } - - @Override - public Expression visit( - io.substrait.expression.Expression.TimestampTZLiteral expr, EmptyVisitationContext context) { - return lit(bldr -> bldr.setNullable(expr.nullable()).setTimestampTz(expr.value())); - } - @Override public Expression visit( io.substrait.expression.Expression.PrecisionTimestampLiteral expr, diff --git a/core/src/main/java/io/substrait/expression/proto/ProtoExpressionConverter.java b/core/src/main/java/io/substrait/expression/proto/ProtoExpressionConverter.java index 3faddc82d..827691a54 100644 --- a/core/src/main/java/io/substrait/expression/proto/ProtoExpressionConverter.java +++ b/core/src/main/java/io/substrait/expression/proto/ProtoExpressionConverter.java @@ -478,10 +478,6 @@ public Expression.Literal from(io.substrait.proto.Expression.Literal literal) { return ExpressionCreator.string(literal.getNullable(), literal.getString()); case BINARY: return ExpressionCreator.binary(literal.getNullable(), literal.getBinary()); - case TIMESTAMP: - return ExpressionCreator.timestamp(literal.getNullable(), literal.getTimestamp()); - case TIMESTAMP_TZ: - return ExpressionCreator.timestampTZ(literal.getNullable(), literal.getTimestampTz()); case PRECISION_TIMESTAMP: return ExpressionCreator.precisionTimestamp( literal.getNullable(), @@ -494,8 +490,6 @@ public Expression.Literal from(io.substrait.proto.Expression.Literal literal) { literal.getPrecisionTimestampTz().getPrecision()); case DATE: return ExpressionCreator.date(literal.getNullable(), literal.getDate()); - case TIME: - return ExpressionCreator.time(literal.getNullable(), literal.getTime()); case PRECISION_TIME: return ExpressionCreator.precisionTime( literal.getNullable(), diff --git a/core/src/main/java/io/substrait/function/ToTypeString.java b/core/src/main/java/io/substrait/function/ToTypeString.java index d3c8c243f..14b662d9c 100644 --- a/core/src/main/java/io/substrait/function/ToTypeString.java +++ b/core/src/main/java/io/substrait/function/ToTypeString.java @@ -76,21 +76,6 @@ public String visit(final Type.Date expr) { return "date"; } - @Override - public String visit(final Type.Time expr) { - return "time"; - } - - @Override - public String visit(final Type.TimestampTZ expr) { - return "tstz"; - } - - @Override - public String visit(final Type.Timestamp expr) { - return "ts"; - } - @Override public String visit(final Type.IntervalYear expr) { return "iyear"; @@ -201,6 +186,11 @@ public String visit(ParameterizedType.IntervalCompound expr) throws RuntimeExcep return "icompound"; } + @Override + public String visit(ParameterizedType.PrecisionTime expr) throws RuntimeException { + return "pt"; + } + @Override public String visit(ParameterizedType.PrecisionTimestamp expr) throws RuntimeException { return "pts"; diff --git a/core/src/main/java/io/substrait/relation/ExpressionCopyOnWriteVisitor.java b/core/src/main/java/io/substrait/relation/ExpressionCopyOnWriteVisitor.java index 4a080c275..16dca0bc6 100644 --- a/core/src/main/java/io/substrait/relation/ExpressionCopyOnWriteVisitor.java +++ b/core/src/main/java/io/substrait/relation/ExpressionCopyOnWriteVisitor.java @@ -114,12 +114,6 @@ public Optional visit(Expression.BinaryLiteral expr, EmptyVisitation return visitLiteral(expr); } - @Override - public Optional visit(Expression.TimeLiteral expr, EmptyVisitationContext context) - throws E { - return visitLiteral(expr); - } - @Override public Optional visit(PrecisionTimeLiteral expr, EmptyVisitationContext context) throws E { @@ -132,18 +126,6 @@ public Optional visit(Expression.DateLiteral expr, EmptyVisitationCo return visitLiteral(expr); } - @Override - public Optional visit( - Expression.TimestampLiteral expr, EmptyVisitationContext context) throws E { - return visitLiteral(expr); - } - - @Override - public Optional visit( - Expression.TimestampTZLiteral expr, EmptyVisitationContext context) throws E { - return visitLiteral(expr); - } - @Override public Optional visit( Expression.PrecisionTimestampLiteral expr, EmptyVisitationContext context) throws E { diff --git a/core/src/main/java/io/substrait/relation/VirtualTableScan.java b/core/src/main/java/io/substrait/relation/VirtualTableScan.java index ce1fb69a3..987bfbee3 100644 --- a/core/src/main/java/io/substrait/relation/VirtualTableScan.java +++ b/core/src/main/java/io/substrait/relation/VirtualTableScan.java @@ -162,21 +162,6 @@ public Integer visit(Type.Date type) throws RuntimeException { return 0; } - @Override - public Integer visit(Type.Time type) throws RuntimeException { - return 0; - } - - @Override - public Integer visit(Type.TimestampTZ type) throws RuntimeException { - return 0; - } - - @Override - public Integer visit(Type.Timestamp type) throws RuntimeException { - return 0; - } - @Override public Integer visit(Type.PrecisionTimestamp type) throws RuntimeException { return 0; diff --git a/core/src/main/java/io/substrait/type/StringTypeVisitor.java b/core/src/main/java/io/substrait/type/StringTypeVisitor.java index 77d8b5c4d..30b130182 100644 --- a/core/src/main/java/io/substrait/type/StringTypeVisitor.java +++ b/core/src/main/java/io/substrait/type/StringTypeVisitor.java @@ -63,21 +63,6 @@ public String visit(Type.Date type) throws RuntimeException { return "date" + n(type); } - @Override - public String visit(Type.Time type) throws RuntimeException { - return "time" + n(type); - } - - @Override - public String visit(Type.TimestampTZ type) throws RuntimeException { - return "timestamp_tz" + n(type); - } - - @Override - public String visit(Type.Timestamp type) throws RuntimeException { - return "timestamp" + n(type); - } - @Override public String visit(Type.IntervalYear type) throws RuntimeException { return "interval_year" + n(type); diff --git a/core/src/main/java/io/substrait/type/Type.java b/core/src/main/java/io/substrait/type/Type.java index 61aecdd5f..65508047b 100644 --- a/core/src/main/java/io/substrait/type/Type.java +++ b/core/src/main/java/io/substrait/type/Type.java @@ -233,73 +233,6 @@ public R accept(final TypeVisitor typeVisitor) th } } - /** - * The time type. - * - * @deprecated use {@link PrecisionTime} instead - */ - @Value.Immutable - @Deprecated - abstract class Time implements Type { - /** - * Creates a builder for {@link Time}. - * - * @return a new builder - */ - public static ImmutableType.Time.Builder builder() { - return ImmutableType.Time.builder(); - } - - @Override - public R accept(final TypeVisitor typeVisitor) throws E { - return typeVisitor.visit(this); - } - } - - /** Deprecated, use {@link PrecisionTimestampTZ} instead */ - @Value.Immutable - @Deprecated - abstract class TimestampTZ implements Type { - - /** - * Creates a builder for {@link TimestampTZ}. - * - * @return a new builder - * @deprecated use {@link PrecisionTimestampTZ#builder()} instead - */ - @Deprecated - public static ImmutableType.TimestampTZ.Builder builder() { - return ImmutableType.TimestampTZ.builder(); - } - - @Override - public R accept(final TypeVisitor typeVisitor) throws E { - return typeVisitor.visit(this); - } - } - - /** Deprecated, use {@link PrecisionTimestamp} instead */ - @Value.Immutable - @Deprecated - abstract class Timestamp implements Type { - - /** - * Creates a builder for {@link Timestamp}. - * - * @return a new builder - * @deprecated use {@link PrecisionTimestamp#builder()} instead - */ - @Deprecated - public static ImmutableType.Timestamp.Builder builder() { - return ImmutableType.Timestamp.builder(); - } - - @Override - public R accept(final TypeVisitor typeVisitor) throws E { - return typeVisitor.visit(this); - } - } - /** The year-month interval type. */ @Value.Immutable abstract class IntervalYear implements Type { diff --git a/core/src/main/java/io/substrait/type/TypeCreator.java b/core/src/main/java/io/substrait/type/TypeCreator.java index 49d7e3678..d095af1b2 100644 --- a/core/src/main/java/io/substrait/type/TypeCreator.java +++ b/core/src/main/java/io/substrait/type/TypeCreator.java @@ -44,18 +44,9 @@ public class TypeCreator { /** The binary type. */ public final Type BINARY; - /** The timestamp (without timezone) type. */ - public final Type TIMESTAMP; - - /** The timestamp with timezone type. */ - public final Type TIMESTAMP_TZ; - /** The date type. */ public final Type DATE; - /** The time type. */ - public final Type TIME; - /** The year-month interval type. */ public final Type INTERVAL_YEAR; @@ -78,10 +69,7 @@ protected TypeCreator(boolean nullable) { FP64 = Type.FP64.builder().nullable(nullable).build(); STRING = Type.Str.builder().nullable(nullable).build(); BINARY = Type.Binary.builder().nullable(nullable).build(); - TIMESTAMP = Type.Timestamp.builder().nullable(nullable).build(); - TIMESTAMP_TZ = Type.TimestampTZ.builder().nullable(nullable).build(); DATE = Type.Date.builder().nullable(nullable).build(); - TIME = Type.Time.builder().nullable(nullable).build(); INTERVAL_YEAR = Type.IntervalYear.builder().nullable(nullable).build(); UUID = Type.UUID.builder().nullable(nullable).build(); } diff --git a/core/src/main/java/io/substrait/type/TypeVisitor.java b/core/src/main/java/io/substrait/type/TypeVisitor.java index ba06dbd49..cadc8c74d 100644 --- a/core/src/main/java/io/substrait/type/TypeVisitor.java +++ b/core/src/main/java/io/substrait/type/TypeVisitor.java @@ -97,39 +97,6 @@ public interface TypeVisitor { */ R visit(Type.Date type) throws E; - /** - * Visits a time type. - * - * @param type the type being visited - * @return the visit result - * @throws E if the visit fails - * @deprecated use {@link #visit(Type.PrecisionTime)} instead - */ - @Deprecated - R visit(Type.Time type) throws E; - - /** - * Visits a timestamp with timezone type. - * - * @param type the type being visited - * @return the visit result - * @throws E if the visit fails - * @deprecated use {@link #visit(Type.PrecisionTimestampTZ)} instead - */ - @Deprecated - R visit(Type.TimestampTZ type) throws E; - - /** - * Visits a timestamp (without timezone) type. - * - * @param type the type being visited - * @return the visit result - * @throws E if the visit fails - * @deprecated use {@link #visit(Type.PrecisionTimestamp)} instead - */ - @Deprecated - R visit(Type.Timestamp type) throws E; - /** * Visits a precision-time type. * @@ -353,21 +320,6 @@ public R visit(Type.Date type) throws E { throw t(); } - @Override - public R visit(Type.Time type) throws E { - throw t(); - } - - @Override - public R visit(Type.TimestampTZ type) throws E { - throw t(); - } - - @Override - public R visit(Type.Timestamp type) throws E { - throw t(); - } - @Override public R visit(Type.IntervalYear type) throws E { throw t(); diff --git a/core/src/main/java/io/substrait/type/parser/ParseToPojo.java b/core/src/main/java/io/substrait/type/parser/ParseToPojo.java index 2079749ef..5140a2aa2 100644 --- a/core/src/main/java/io/substrait/type/parser/ParseToPojo.java +++ b/core/src/main/java/io/substrait/type/parser/ParseToPojo.java @@ -182,26 +182,11 @@ public Type visitBinary(final SubstraitTypeParser.BinaryContext ctx) { return withNull(ctx).BINARY; } - @Override - public Type visitTimestamp(final SubstraitTypeParser.TimestampContext ctx) { - return withNull(ctx).TIMESTAMP; - } - - @Override - public Type visitTimestampTz(final SubstraitTypeParser.TimestampTzContext ctx) { - return withNull(ctx).TIMESTAMP_TZ; - } - @Override public Type visitDate(final SubstraitTypeParser.DateContext ctx) { return withNull(ctx).DATE; } - @Override - public Type visitTime(final SubstraitTypeParser.TimeContext ctx) { - return withNull(ctx).TIME; - } - @Override public Type visitIntervalYear(final SubstraitTypeParser.IntervalYearContext ctx) { return withNull(ctx).INTERVAL_YEAR; @@ -702,8 +687,22 @@ public TypeExpression visitAnyType(SubstraitTypeParser.AnyTypeContext anyType) { @Override public TypeExpression visitFunctionCall(final SubstraitTypeParser.FunctionCallContext ctx) { checkExpression(); + String functionName = ctx.Identifier().getText(); + if (functionName.equalsIgnoreCase("integer_parameter")) { + // integer_parameter(argName) coerces a value argument to an integer type parameter. + // Represent the result as a StringLiteral (parameter reference) of the argument name. + if (ctx.expr().size() != 1) { + throw new IllegalStateException( + "integer_parameter requires exactly one argument, got: " + ctx.expr().size()); + } + return ctx.expr(0).accept(this); + } if (ctx.expr().size() != 2) { - throw new IllegalStateException("Only two argument functions exist for type expressions."); + throw new IllegalStateException( + "Only two argument functions exist for type expressions, got " + + ctx.expr().size() + + " for: " + + functionName); } TypeExpression.BinaryOperation.OpType type = getFunctionType(ctx.Identifier().getSymbol()); return TypeExpression.BinaryOperation.builder() diff --git a/core/src/main/java/io/substrait/type/proto/BaseProtoConverter.java b/core/src/main/java/io/substrait/type/proto/BaseProtoConverter.java index 30823e745..55a28e4a8 100644 --- a/core/src/main/java/io/substrait/type/proto/BaseProtoConverter.java +++ b/core/src/main/java/io/substrait/type/proto/BaseProtoConverter.java @@ -86,21 +86,6 @@ public final T visit(final Type.Date expr) { return typeContainer(expr).DATE; } - @Override - public final T visit(final Type.Time expr) { - return typeContainer(expr).TIME; - } - - @Override - public final T visit(final Type.TimestampTZ expr) { - return typeContainer(expr).TIMESTAMP_TZ; - } - - @Override - public final T visit(final Type.Timestamp expr) { - return typeContainer(expr).TIMESTAMP; - } - @Override public final T visit(final Type.IntervalYear expr) { return typeContainer(expr).INTERVAL_YEAR; diff --git a/core/src/main/java/io/substrait/type/proto/BaseProtoTypes.java b/core/src/main/java/io/substrait/type/proto/BaseProtoTypes.java index c6490c9b1..5146ddd64 100644 --- a/core/src/main/java/io/substrait/type/proto/BaseProtoTypes.java +++ b/core/src/main/java/io/substrait/type/proto/BaseProtoTypes.java @@ -15,10 +15,7 @@ abstract class BaseProtoTypes { public final T FP64; public final T STRING; public final T BINARY; - public final T TIMESTAMP; - public final T TIMESTAMP_TZ; public final T DATE; - public final T TIME; public final T INTERVAL_YEAR; public final T UUID; @@ -33,10 +30,7 @@ public BaseProtoTypes(Type.Nullability nullability) { FP64 = wrap(Type.FP64.newBuilder().setNullability(nullability).build()); STRING = wrap(Type.String.newBuilder().setNullability(nullability).build()); BINARY = wrap(Type.Binary.newBuilder().setNullability(nullability).build()); - TIMESTAMP = wrap(Type.Timestamp.newBuilder().setNullability(nullability).build()); - TIMESTAMP_TZ = wrap(Type.TimestampTZ.newBuilder().setNullability(nullability).build()); DATE = wrap(Type.Date.newBuilder().setNullability(nullability).build()); - TIME = wrap(Type.Time.newBuilder().setNullability(nullability).build()); INTERVAL_YEAR = wrap(Type.IntervalYear.newBuilder().setNullability(nullability).build()); UUID = wrap(Type.UUID.newBuilder().setNullability(nullability).build()); } diff --git a/core/src/main/java/io/substrait/type/proto/ProtoTypeConverter.java b/core/src/main/java/io/substrait/type/proto/ProtoTypeConverter.java index bfa642e42..425c76a4d 100644 --- a/core/src/main/java/io/substrait/type/proto/ProtoTypeConverter.java +++ b/core/src/main/java/io/substrait/type/proto/ProtoTypeConverter.java @@ -50,12 +50,8 @@ public Type from(io.substrait.proto.Type type) { return n(type.getString().getNullability()).STRING; case BINARY: return n(type.getBinary().getNullability()).BINARY; - case TIMESTAMP: - return n(type.getTimestamp().getNullability()).TIMESTAMP; case DATE: return n(type.getDate().getNullability()).DATE; - case TIME: - return n(type.getTime().getNullability()).TIME; case INTERVAL_YEAR: return n(type.getIntervalYear().getNullability()).INTERVAL_YEAR; case INTERVAL_DAY: @@ -66,8 +62,6 @@ public Type from(io.substrait.proto.Type type) { case INTERVAL_COMPOUND: return n(type.getIntervalCompound().getNullability()) .intervalCompound(type.getIntervalCompound().getPrecision()); - case TIMESTAMP_TZ: - return n(type.getTimestampTz().getNullability()).TIMESTAMP_TZ; case UUID: return n(type.getUuid().getNullability()).UUID; case FIXED_CHAR: diff --git a/core/src/main/java/io/substrait/type/proto/TypeProtoConverter.java b/core/src/main/java/io/substrait/type/proto/TypeProtoConverter.java index 5d4a1be4a..6c22a1a9e 100644 --- a/core/src/main/java/io/substrait/type/proto/TypeProtoConverter.java +++ b/core/src/main/java/io/substrait/type/proto/TypeProtoConverter.java @@ -236,14 +236,8 @@ protected Type wrap(final Object o) { return bldr.setString((Type.String) o).build(); } else if (o instanceof Type.Binary) { return bldr.setBinary((Type.Binary) o).build(); - } else if (o instanceof Type.Timestamp) { - return bldr.setTimestamp((Type.Timestamp) o).build(); } else if (o instanceof Type.Date) { return bldr.setDate((Type.Date) o).build(); - } else if (o instanceof Type.Time) { - return bldr.setTime((Type.Time) o).build(); - } else if (o instanceof Type.TimestampTZ) { - return bldr.setTimestampTz((Type.TimestampTZ) o).build(); } else if (o instanceof Type.IntervalYear) { return bldr.setIntervalYear((Type.IntervalYear) o).build(); } else if (o instanceof Type.IntervalDay) { diff --git a/core/src/test/java/io/substrait/function/ToTypeStringTest.java b/core/src/test/java/io/substrait/function/ToTypeStringTest.java index 02f81411a..347825aee 100644 --- a/core/src/test/java/io/substrait/function/ToTypeStringTest.java +++ b/core/src/test/java/io/substrait/function/ToTypeStringTest.java @@ -27,9 +27,6 @@ static Stream types() { Arguments.of(c.STRING, "str"), Arguments.of(c.BINARY, "vbin"), Arguments.of(c.DATE, "date"), - Arguments.of(c.TIME, "time"), - Arguments.of(c.TIMESTAMP, "ts"), - Arguments.of(c.TIMESTAMP_TZ, "tstz"), Arguments.of(c.INTERVAL_YEAR, "iyear"), Arguments.of(c.UUID, "uuid"), Arguments.of(c.fixedChar(1), "fchar"), diff --git a/core/src/test/java/io/substrait/type/proto/DynamicParameterRoundtripTest.java b/core/src/test/java/io/substrait/type/proto/DynamicParameterRoundtripTest.java index 3c3b99ed5..a41946ffc 100644 --- a/core/src/test/java/io/substrait/type/proto/DynamicParameterRoundtripTest.java +++ b/core/src/test/java/io/substrait/type/proto/DynamicParameterRoundtripTest.java @@ -98,11 +98,11 @@ void dynamicParameterDecimal() { void dynamicParameterTimestamp() { Expression.DynamicParameter dp = Expression.DynamicParameter.builder() - .type(TypeCreator.NULLABLE.TIMESTAMP) + .type(TypeCreator.NULLABLE.precisionTimestamp(6)) .parameterReference(7) .build(); - assertDynamicParameter(dp, TypeCreator.NULLABLE.TIMESTAMP, 7); + assertDynamicParameter(dp, TypeCreator.NULLABLE.precisionTimestamp(6), 7); verifyRoundTrip(dp); } diff --git a/core/src/test/java/io/substrait/type/proto/SortRelRoundtripTest.java b/core/src/test/java/io/substrait/type/proto/SortRelRoundtripTest.java index 039062f24..d3bebd41d 100644 --- a/core/src/test/java/io/substrait/type/proto/SortRelRoundtripTest.java +++ b/core/src/test/java/io/substrait/type/proto/SortRelRoundtripTest.java @@ -14,7 +14,7 @@ class SortRelRoundtripTest extends TestBase { sb.namedScan( Collections.singletonList("test_table"), Arrays.asList("id", "amount", "name", "category", "timestamp"), - Arrays.asList(R.I64, R.FP64, R.STRING, R.STRING, R.TIMESTAMP)); + Arrays.asList(R.I64, R.FP64, R.STRING, R.STRING, R.precisionTimestamp(6))); @Test void simpleSortAscending() { diff --git a/core/src/test/java/io/substrait/type/proto/TestTypeRoundtrip.java b/core/src/test/java/io/substrait/type/proto/TestTypeRoundtrip.java index 5502590c6..fff65ac37 100644 --- a/core/src/test/java/io/substrait/type/proto/TestTypeRoundtrip.java +++ b/core/src/test/java/io/substrait/type/proto/TestTypeRoundtrip.java @@ -33,10 +33,7 @@ static Stream types() { creator.FP64, creator.STRING, creator.BINARY, - creator.TIME, creator.DATE, - creator.TIMESTAMP, - creator.TIMESTAMP_TZ, creator.INTERVAL_YEAR, creator.UUID, creator.fixedChar(25), @@ -49,8 +46,11 @@ static Stream types() { creator.precisionTimestamp(1), creator.precisionTimestampTZ(2), creator.map(creator.I8, creator.I16), - creator.list(creator.TIME), - creator.struct(creator.TIME, creator.TIMESTAMP, creator.TIMESTAMP_TZ))); + creator.list(creator.DATE), + creator.struct( + creator.DATE, + creator.precisionTimestamp(6), + creator.precisionTimestampTZ(6)))); } @ParameterizedTest diff --git a/examples/substrait-spark/src/main/java/io/substrait/examples/util/ExpressionStringify.java b/examples/substrait-spark/src/main/java/io/substrait/examples/util/ExpressionStringify.java index 1cb662b70..4d27f2534 100644 --- a/examples/substrait-spark/src/main/java/io/substrait/examples/util/ExpressionStringify.java +++ b/examples/substrait-spark/src/main/java/io/substrait/examples/util/ExpressionStringify.java @@ -35,9 +35,6 @@ import io.substrait.expression.Expression.StrLiteral; import io.substrait.expression.Expression.StructLiteral; import io.substrait.expression.Expression.Switch; -import io.substrait.expression.Expression.TimeLiteral; -import io.substrait.expression.Expression.TimestampLiteral; -import io.substrait.expression.Expression.TimestampTZLiteral; import io.substrait.expression.Expression.UUIDLiteral; import io.substrait.expression.Expression.UserDefinedAnyLiteral; import io.substrait.expression.Expression.UserDefinedStructLiteral; @@ -113,11 +110,6 @@ public String visit(BinaryLiteral expr, EmptyVisitationContext context) throws R return ""; } - @Override - public String visit(TimeLiteral expr, EmptyVisitationContext context) throws RuntimeException { - return ""; - } - @Override public String visit(PrecisionTimeLiteral expr, EmptyVisitationContext context) throws RuntimeException { @@ -129,18 +121,6 @@ public String visit(DateLiteral expr, EmptyVisitationContext context) throws Run return ""; } - @Override - public String visit(TimestampLiteral expr, EmptyVisitationContext context) - throws RuntimeException { - return ""; - } - - @Override - public String visit(TimestampTZLiteral expr, EmptyVisitationContext context) - throws RuntimeException { - return ""; - } - @Override public String visit(IntervalYearLiteral expr, EmptyVisitationContext context) throws RuntimeException { diff --git a/examples/substrait-spark/src/main/java/io/substrait/examples/util/TypeStringify.java b/examples/substrait-spark/src/main/java/io/substrait/examples/util/TypeStringify.java index 21d9f1524..11db9aa1e 100644 --- a/examples/substrait-spark/src/main/java/io/substrait/examples/util/TypeStringify.java +++ b/examples/substrait-spark/src/main/java/io/substrait/examples/util/TypeStringify.java @@ -21,9 +21,6 @@ import io.substrait.type.Type.PrecisionTime; import io.substrait.type.Type.Str; import io.substrait.type.Type.Struct; -import io.substrait.type.Type.Time; -import io.substrait.type.Type.Timestamp; -import io.substrait.type.Type.TimestampTZ; import io.substrait.type.Type.UUID; import io.substrait.type.Type.UserDefined; import io.substrait.type.Type.VarChar; @@ -92,23 +89,6 @@ public String visit(Date type) throws RuntimeException { return type.getClass().getSimpleName(); } - @Override - public String visit(Time type) throws RuntimeException { - return type.getClass().getSimpleName(); - } - - @Override - @Deprecated - public String visit(TimestampTZ type) throws RuntimeException { - return type.getClass().getSimpleName(); - } - - @Override - @Deprecated - public String visit(Timestamp type) throws RuntimeException { - return type.getClass().getSimpleName(); - } - @Override public String visit(Type.PrecisionTimestamp type) throws RuntimeException { return type.getClass().getSimpleName(); diff --git a/isthmus/src/main/java/io/substrait/isthmus/SimpleExtensionToSqlOperator.java b/isthmus/src/main/java/io/substrait/isthmus/SimpleExtensionToSqlOperator.java index 25a4009a4..1f56f0d52 100644 --- a/isthmus/src/main/java/io/substrait/isthmus/SimpleExtensionToSqlOperator.java +++ b/isthmus/src/main/java/io/substrait/isthmus/SimpleExtensionToSqlOperator.java @@ -466,21 +466,6 @@ public SqlTypeName visit(Type.Date expr) { return SqlTypeName.DATE; } - @Override - public SqlTypeName visit(Type.Time expr) { - return SqlTypeName.TIME; - } - - @Override - public SqlTypeName visit(Type.TimestampTZ expr) { - return SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE; - } - - @Override - public SqlTypeName visit(Type.Timestamp expr) { - return SqlTypeName.TIMESTAMP; - } - @Override public SqlTypeName visit(Type.IntervalYear year) { return SqlTypeName.INTERVAL_YEAR_MONTH; diff --git a/isthmus/src/main/java/io/substrait/isthmus/TypeConverter.java b/isthmus/src/main/java/io/substrait/isthmus/TypeConverter.java index 000032cac..add54572d 100644 --- a/isthmus/src/main/java/io/substrait/isthmus/TypeConverter.java +++ b/isthmus/src/main/java/io/substrait/isthmus/TypeConverter.java @@ -297,21 +297,6 @@ public RelDataType visit(Type.Date expr) { return t(n(expr), SqlTypeName.DATE); } - @Override - public RelDataType visit(Type.Time expr) { - return t(n(expr), SqlTypeName.TIME, 6); - } - - @Override - public RelDataType visit(Type.TimestampTZ expr) { - return t(n(expr), SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, 6); - } - - @Override - public RelDataType visit(Type.Timestamp expr) { - return t(n(expr), SqlTypeName.TIMESTAMP, 6); - } - @Override public RelDataType visit(Type.PrecisionTime expr) { int maxPrecision = typeFactory.getTypeSystem().getMaxPrecision(SqlTypeName.TIME); diff --git a/isthmus/src/main/java/io/substrait/isthmus/expression/EnumConverter.java b/isthmus/src/main/java/io/substrait/isthmus/expression/EnumConverter.java index 2f95ca276..8164af44c 100644 --- a/isthmus/src/main/java/io/substrait/isthmus/expression/EnumConverter.java +++ b/isthmus/src/main/java/io/substrait/isthmus/expression/EnumConverter.java @@ -32,15 +32,9 @@ public class EnumConverter { private static final Map>> calciteEnumMap = new HashMap<>(); static { - // deprecated {@link io.substrait.type.Type.Timestamp} calciteEnumMap.put( - argAnchor(DefaultExtensionCatalog.FUNCTIONS_DATETIME, "extract:req_ts", 0), + argAnchor(DefaultExtensionCatalog.FUNCTIONS_DATETIME, "extract:req_pt", 0), TimeUnitRange.class); - // deprecated {@link io.substrait.type.Type.TimestampTZ} - calciteEnumMap.put( - argAnchor(DefaultExtensionCatalog.FUNCTIONS_DATETIME, "extract:req_tstz_str", 0), - TimeUnitRange.class); - calciteEnumMap.put( argAnchor(DefaultExtensionCatalog.FUNCTIONS_DATETIME, "extract:req_pts", 0), TimeUnitRange.class); @@ -50,9 +44,6 @@ public class EnumConverter { calciteEnumMap.put( argAnchor(DefaultExtensionCatalog.FUNCTIONS_DATETIME, "extract:req_date", 0), TimeUnitRange.class); - calciteEnumMap.put( - argAnchor(DefaultExtensionCatalog.FUNCTIONS_DATETIME, "extract:req_time", 0), - TimeUnitRange.class); calciteEnumMap.put( argAnchor(DefaultExtensionCatalog.FUNCTIONS_STRING, "trim:vchar_vchar", 0), diff --git a/isthmus/src/main/java/io/substrait/isthmus/expression/ExpressionRexConverter.java b/isthmus/src/main/java/io/substrait/isthmus/expression/ExpressionRexConverter.java index 925b17b35..a7ace5d46 100644 --- a/isthmus/src/main/java/io/substrait/isthmus/expression/ExpressionRexConverter.java +++ b/isthmus/src/main/java/io/substrait/isthmus/expression/ExpressionRexConverter.java @@ -14,7 +14,6 @@ import io.substrait.expression.Expression.SetPredicate; import io.substrait.expression.Expression.SingleOrList; import io.substrait.expression.Expression.Switch; -import io.substrait.expression.Expression.TimestampTZLiteral; import io.substrait.expression.FieldReference; import io.substrait.expression.FieldReference.ReferenceSegment; import io.substrait.expression.FunctionArg; @@ -232,12 +231,6 @@ public RexNode visit(Expression.BinaryLiteral expr, Context context) throws Runt true); } - @Override - public RexNode visit(Expression.TimeLiteral expr, Context context) throws RuntimeException { - return rexBuilder.makeLiteral( - createTimeString(expr.value(), 6), typeConverter.toCalcite(typeFactory, expr.getType())); - } - @Override public RexNode visit(PrecisionTimeLiteral expr, Context context) throws RuntimeException { int maxPrecision = typeFactory.getTypeSystem().getMaxPrecision(SqlTypeName.TIME); @@ -316,18 +309,6 @@ public RexNode visit(Expression.DateLiteral expr, Context context) throws Runtim expr.value(), typeConverter.toCalcite(typeFactory, expr.getType())); } - @Override - public RexNode visit(Expression.TimestampLiteral expr, Context context) throws RuntimeException { - return rexBuilder.makeLiteral( - getTimestampString(expr.value()), typeConverter.toCalcite(typeFactory, expr.getType())); - } - - @Override - public RexNode visit(TimestampTZLiteral expr, Context context) throws RuntimeException { - return rexBuilder.makeLiteral( - getTimestampString(expr.value()), typeConverter.toCalcite(typeFactory, expr.getType())); - } - @Override public RexNode visit(PrecisionTimestampLiteral expr, Context context) throws RuntimeException { int maxPrecision = typeFactory.getTypeSystem().getMaxPrecision(SqlTypeName.TIMESTAMP); @@ -357,10 +338,6 @@ public RexNode visit(PrecisionTimestampTZLiteral expr, Context context) throws R typeConverter.toCalcite(typeFactory, expr.getType())); } - private TimestampString getTimestampString(long microSec) { - return getTimestampString(microSec, 6); - } - private TimestampString getTimestampString(long value, int precision) { switch (precision) { case 0: diff --git a/isthmus/src/main/java/io/substrait/isthmus/expression/IgnoreNullableAndParameters.java b/isthmus/src/main/java/io/substrait/isthmus/expression/IgnoreNullableAndParameters.java index c69617541..378a6bcb5 100644 --- a/isthmus/src/main/java/io/substrait/isthmus/expression/IgnoreNullableAndParameters.java +++ b/isthmus/src/main/java/io/substrait/isthmus/expression/IgnoreNullableAndParameters.java @@ -140,39 +140,6 @@ public Boolean visit(Type.Date type) { return typeToMatch instanceof Type.Date; } - /** - * Compares {@link Type.Time} ignoring nullability. - * - * @param type time type - * @return {@code true} if {@code typeToMatch} is {@link Type.Time} - */ - @Override - public Boolean visit(Type.Time type) { - return typeToMatch instanceof Type.Time; - } - - /** - * Compares {@link Type.TimestampTZ} ignoring nullability. - * - * @param type timestamp-with-time-zone type - * @return {@code true} if {@code typeToMatch} is {@link Type.TimestampTZ} - */ - @Override - public Boolean visit(Type.TimestampTZ type) { - return typeToMatch instanceof Type.TimestampTZ; - } - - /** - * Compares {@link Type.Timestamp} ignoring nullability. - * - * @param type timestamp type - * @return {@code true} if {@code typeToMatch} is {@link Type.Timestamp} - */ - @Override - public Boolean visit(Type.Timestamp type) { - return typeToMatch instanceof Type.Timestamp; - } - /** * Compares {@link Type.IntervalYear} ignoring nullability. * diff --git a/isthmus/src/test/java/io/substrait/isthmus/FunctionConversionTest.java b/isthmus/src/test/java/io/substrait/isthmus/FunctionConversionTest.java index ff9cb3ab5..5e4e80ee2 100644 --- a/isthmus/src/test/java/io/substrait/isthmus/FunctionConversionTest.java +++ b/isthmus/src/test/java/io/substrait/isthmus/FunctionConversionTest.java @@ -15,16 +15,12 @@ import io.substrait.isthmus.expression.RexExpressionConverter; import io.substrait.isthmus.expression.ScalarFunctionConverter; import io.substrait.isthmus.expression.WindowFunctionConverter; -import io.substrait.type.Type; import io.substrait.type.TypeCreator; import java.util.stream.Stream; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; /** * Verify that "problematic" Substrait functions can be converted to Calcite and back successfully @@ -74,27 +70,6 @@ void subtractDateIDay() { assertEquals(expr, reverse); } - @Test - void extractTimestampTzScalarFunction() { - ScalarFunctionInvocation reqTstzFn = - sb.scalarFn( - DefaultExtensionCatalog.FUNCTIONS_DATETIME, - "extract:req_tstz_str", - TypeCreator.REQUIRED.I64, - EnumArg.builder().value("MONTH").build(), - Expression.TimestampTZLiteral.builder().value(0).build(), - Expression.StrLiteral.builder().value("GMT").build()); - - RexNode calciteExpr = reqTstzFn.accept(expressionRexConverter, Context.newContext()); - assertEquals(SqlKind.EXTRACT, calciteExpr.getKind()); - assertInstanceOf(RexCall.class, calciteExpr); - - RexCall extract = (RexCall) calciteExpr; - assertEquals( - "EXTRACT(FLAG(MONTH), 1970-01-01 00:00:00:TIMESTAMP_WITH_LOCAL_TIME_ZONE(6), 'GMT':VARCHAR)", - extract.toString()); - } - @Test void extractPrecisionTimestampTzScalarFunction() { ScalarFunctionInvocation reqPtstzFn = @@ -116,24 +91,6 @@ void extractPrecisionTimestampTzScalarFunction() { extract.toString()); } - @Test - void extractTimestampScalarFunction() { - ScalarFunctionInvocation reqTsFn = - sb.scalarFn( - DefaultExtensionCatalog.FUNCTIONS_DATETIME, - "extract:req_ts", - TypeCreator.REQUIRED.I64, - EnumArg.builder().value("MONTH").build(), - Expression.TimestampLiteral.builder().value(0).build()); - - RexNode calciteExpr = reqTsFn.accept(expressionRexConverter, Context.newContext()); - assertEquals(SqlKind.EXTRACT, calciteExpr.getKind()); - assertInstanceOf(RexCall.class, calciteExpr); - - RexCall extract = (RexCall) calciteExpr; - assertEquals("EXTRACT(FLAG(MONTH), 1970-01-01 00:00:00:TIMESTAMP(6))", extract.toString()); - } - @Test void extractPrecisionTimestampScalarFunction() { ScalarFunctionInvocation reqPtsFn = @@ -175,10 +132,10 @@ void extractTimeScalarFunction() { ScalarFunctionInvocation reqTimeFn = sb.scalarFn( DefaultExtensionCatalog.FUNCTIONS_DATETIME, - "extract:req_time", + "extract:req_pt", TypeCreator.REQUIRED.I64, EnumArg.builder().value("MINUTE").build(), - Expression.TimeLiteral.builder().value(0).build()); + Expression.PrecisionTimeLiteral.builder().value(0).precision(6).build()); RexNode calciteExpr = reqTimeFn.accept(expressionRexConverter, Context.newContext()); assertEquals(SqlKind.EXTRACT, calciteExpr.getKind()); @@ -207,23 +164,6 @@ void extractDateWithIndexing() { assertEquals("EXTRACT(FLAG(MONTH), 1970-01-01)", extract.toString()); } - @Test - void unsupportedExtractTimestampTzWithIndexing() { - ScalarFunctionInvocation reqReqTstzFn = - sb.scalarFn( - DefaultExtensionCatalog.FUNCTIONS_DATETIME, - "extract:req_req_tstz_str", - TypeCreator.REQUIRED.I64, - EnumArg.builder().value("MONTH").build(), - EnumArg.builder().value("ONE").build(), - Expression.TimestampTZLiteral.builder().value(0).build(), - Expression.StrLiteral.builder().value("GMT").build()); - - assertThrows( - UnsupportedOperationException.class, - () -> reqReqTstzFn.accept(expressionRexConverter, Context.newContext())); - } - @Test void unsupportedExtractPrecisionTimestampTzWithIndexing() { ScalarFunctionInvocation reqReqPtstzFn = @@ -241,22 +181,6 @@ void unsupportedExtractPrecisionTimestampTzWithIndexing() { () -> reqReqPtstzFn.accept(expressionRexConverter, Context.newContext())); } - @Test - void unsupportedExtractTimestampWithIndexing() { - ScalarFunctionInvocation reqReqTsFn = - sb.scalarFn( - DefaultExtensionCatalog.FUNCTIONS_DATETIME, - "extract:req_req_ts", - TypeCreator.REQUIRED.I64, - EnumArg.builder().value("MONTH").build(), - EnumArg.builder().value("ONE").build(), - Expression.TimestampLiteral.builder().value(0).build()); - - assertThrows( - UnsupportedOperationException.class, - () -> reqReqTsFn.accept(expressionRexConverter, Context.newContext())); - } - @Test void unsupportedExtractPrecisionTimestampWithIndexing() { ScalarFunctionInvocation reqReqPtsFn = @@ -288,49 +212,62 @@ void concatStringLiteralAndChar() throws Exception { assertProtoPlanRoundrip("select 'brand_'||P_BRAND from PART"); } - /** - * Provides test cases for strptime function tests. - * - * @return Stream of test arguments containing: function name, input string value, format string, - * output type, and expected Calcite function name - */ - private static Stream strptimeTestCases() { - return Stream.of( - Arguments.of( - "strptime_time:str_str", - "12:34:56", - "%H:%M:%S", + @Test + void strptimeTime() { + Expression.StrLiteral inputString = Expression.StrLiteral.builder().value("12:34:56").build(); + Expression.StrLiteral formatString = Expression.StrLiteral.builder().value("%H:%M:%S").build(); + Expression.I8Literal precision = ExpressionCreator.i8(false, (byte) 6); + ScalarFunctionInvocation strptimeFn = + sb.scalarFn( + DefaultExtensionCatalog.FUNCTIONS_DATETIME, + "strptime_time:str_str_i8", TypeCreator.REQUIRED.precisionTime(6), - "PARSE_TIME"), - Arguments.of( - "strptime_timestamp:str_str", - "2026-01-29T12:34:56", - "%Y:%m:%dT%H:%M:%S", + inputString, + formatString, + precision); + + // tests Substrait -> Calcite + RexNode calciteExpr = strptimeFn.accept(expressionRexConverter, Context.newContext()); + assertEquals(SqlKind.OTHER_FUNCTION, calciteExpr.getKind()); + assertInstanceOf(RexCall.class, calciteExpr); + assertEquals( + "PARSE_TIME('%H:%M:%S':VARCHAR, '12:34:56':VARCHAR, 6:TINYINT)", calciteExpr.toString()); + } + + @Test + void strptimeTimestamp() { + Expression.StrLiteral inputString = + Expression.StrLiteral.builder().value("2026-01-29T12:34:56").build(); + Expression.StrLiteral formatString = + Expression.StrLiteral.builder().value("%Y:%m:%dT%H:%M:%S").build(); + Expression.I8Literal precision = ExpressionCreator.i8(false, (byte) 6); + ScalarFunctionInvocation strptimeFn = + sb.scalarFn( + DefaultExtensionCatalog.FUNCTIONS_DATETIME, + "strptime_timestamp:str_str_i8", TypeCreator.REQUIRED.precisionTimestamp(6), - "PARSE_TIMESTAMP"), - Arguments.of( - "strptime_date:str_str", - "2026-01-29", - "%Y:%m:%d", - TypeCreator.REQUIRED.DATE, - "PARSE_DATE")); + inputString, + formatString, + precision); + + // tests Substrait -> Calcite + RexNode calciteExpr = strptimeFn.accept(expressionRexConverter, Context.newContext()); + assertEquals(SqlKind.OTHER_FUNCTION, calciteExpr.getKind()); + assertInstanceOf(RexCall.class, calciteExpr); + assertEquals( + "PARSE_TIMESTAMP('%Y:%m:%dT%H:%M:%S':VARCHAR, '2026-01-29T12:34:56':VARCHAR, 6:TINYINT)", + calciteExpr.toString()); } - @ParameterizedTest - @MethodSource("strptimeTestCases") - void testStrptimeFunctions( - String functionSignature, - String inputValue, - String formatValue, - Type outputType, - String expectedCalciteFunctionName) { - Expression.StrLiteral inputString = Expression.StrLiteral.builder().value(inputValue).build(); - Expression.StrLiteral formatString = Expression.StrLiteral.builder().value(formatValue).build(); + @Test + void strptimeDate() { + Expression.StrLiteral inputString = Expression.StrLiteral.builder().value("2026-01-29").build(); + Expression.StrLiteral formatString = Expression.StrLiteral.builder().value("%Y:%m:%d").build(); ScalarFunctionInvocation strptimeFn = sb.scalarFn( DefaultExtensionCatalog.FUNCTIONS_DATETIME, - functionSignature, - outputType, + "strptime_date:str_str", + TypeCreator.REQUIRED.DATE, inputString, formatString); @@ -338,11 +275,7 @@ void testStrptimeFunctions( RexNode calciteExpr = strptimeFn.accept(expressionRexConverter, Context.newContext()); assertEquals(SqlKind.OTHER_FUNCTION, calciteExpr.getKind()); assertInstanceOf(RexCall.class, calciteExpr); - - String expectedCallString = - String.format( - "%s('%s':VARCHAR, '%s':VARCHAR)", expectedCalciteFunctionName, formatValue, inputValue); - assertEquals(expectedCallString, calciteExpr.toString()); + assertEquals("PARSE_DATE('%Y:%m:%d':VARCHAR, '2026-01-29':VARCHAR)", calciteExpr.toString()); // tests the reverse Calcite -> Substrait Expression reverse = calciteExpr.accept(rexExpressionConverter); diff --git a/spark/spark_dialect.yaml b/spark/spark_dialect.yaml index 9bcfd5e6a..52c9cb47a 100644 --- a/spark/spark_dialect.yaml +++ b/spark/spark_dialect.yaml @@ -486,8 +486,6 @@ supported_scalar_functions: - "iyear_iyear" - "pts_pts" - "ptstz_ptstz" - - "ts_ts" - - "tstz_tstz" - source: "datetime" name: "gte" system_metadata: @@ -499,8 +497,6 @@ supported_scalar_functions: - "iyear_iyear" - "pts_pts" - "ptstz_ptstz" - - "ts_ts" - - "tstz_tstz" - source: "datetime" name: "lt" system_metadata: @@ -512,8 +508,6 @@ supported_scalar_functions: - "iyear_iyear" - "pts_pts" - "ptstz_ptstz" - - "ts_ts" - - "tstz_tstz" - source: "datetime" name: "lte" system_metadata: @@ -525,8 +519,6 @@ supported_scalar_functions: - "iyear_iyear" - "pts_pts" - "ptstz_ptstz" - - "ts_ts" - - "tstz_tstz" - source: "logarithmic" name: "ln" system_metadata: @@ -798,11 +790,9 @@ supported_aggregate_functions: - "date" - "iday" - "iyear" + - "pt" - "pts" - "ptstz" - - "time" - - "ts" - - "tstz" - source: "datetime" name: "min" system_metadata: @@ -812,11 +802,9 @@ supported_aggregate_functions: - "date" - "iday" - "iyear" + - "pt" - "pts" - "ptstz" - - "time" - - "ts" - - "tstz" supported_window_functions: - source: "arithmetic" name: "cume_dist" diff --git a/spark/src/main/scala/io/substrait/spark/expression/IgnoreNullableAndParameters.scala b/spark/src/main/scala/io/substrait/spark/expression/IgnoreNullableAndParameters.scala index 0db03388e..c087832a0 100644 --- a/spark/src/main/scala/io/substrait/spark/expression/IgnoreNullableAndParameters.scala +++ b/spark/src/main/scala/io/substrait/spark/expression/IgnoreNullableAndParameters.scala @@ -44,15 +44,6 @@ class IgnoreNullableAndParameters(val typeToMatch: ParameterizedType) override def visit(`type`: Type.Date): Boolean = typeToMatch.isInstanceOf[Type.Date] - @nowarn - override def visit(`type`: Type.Time): Boolean = typeToMatch.isInstanceOf[Type.Time] - - @nowarn - override def visit(`type`: Type.TimestampTZ): Boolean = typeToMatch.isInstanceOf[Type.TimestampTZ] - - @nowarn - override def visit(`type`: Type.Timestamp): Boolean = typeToMatch.isInstanceOf[Type.Timestamp] - override def visit(`type`: Type.IntervalYear): Boolean = typeToMatch.isInstanceOf[Type.IntervalYear] diff --git a/substrait b/substrait index 7cceb8369..f09071699 160000 --- a/substrait +++ b/substrait @@ -1 +1 @@ -Subproject commit 7cceb8369cd533134431979835124de5634a4a47 +Subproject commit f09071699edcb41d90e1080625e9ee25fce2b81c