diff --git a/phoenix-core-client/src/main/antlr3/PhoenixSQL.g b/phoenix-core-client/src/main/antlr3/PhoenixSQL.g index bb7d3d8a89b..768eb602491 100644 --- a/phoenix-core-client/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core-client/src/main/antlr3/PhoenixSQL.g @@ -203,6 +203,7 @@ import java.lang.Boolean; import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.Set; import java.util.Stack; @@ -233,7 +234,7 @@ import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.parse.LikeParseNode.LikeType; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.parse.AddJarsStatement; -import org.apache.phoenix.parse.ExplainType; +import org.apache.phoenix.parse.ExplainOptions; } @lexer::header { @@ -318,6 +319,69 @@ package org.apache.phoenix.parse; } } + /** + * Closed-set keys for the EXPLAIN option list, used to reject duplicate options as they are + * encountered while parsing. + */ + private enum ExplainOpt { REGIONS, VERBOSE, FORMAT } + + /** + * Mutable accumulator for the EXPLAIN option list. Lifetime is one explain_node parse. + */ + private static final class ExplainOptsAcc { + boolean regions; + boolean verbose; + ExplainOptions.Format format; + final EnumSet seen = EnumSet.noneOf(ExplainOpt.class); + + void mark(ExplainOpt opt) { + if (!seen.add(opt)) { + throw new RuntimeException("Duplicate EXPLAIN option: " + opt); + } + } + + ExplainOptions build() { + return new ExplainOptions(regions, verbose, + format == null ? ExplainOptions.Format.TEXT : format); + } + } + + /** + * Parse a single EXPLAIN option into the given accumulator. The option name is matched + * case-insensitively against the closed set {REGIONS, VERBOSE, FORMAT}. REGIONS and VERBOSE are + * flags and must not carry a value; FORMAT requires a value of TEXT or JSON. Duplicate options + * are rejected. + */ + private void parseExplainOption(ExplainOptsAcc opts, String name, String value) { + if ("REGIONS".equalsIgnoreCase(name)) { + if (value != null) { + throw new RuntimeException("EXPLAIN option REGIONS does not take a value"); + } + opts.mark(ExplainOpt.REGIONS); + opts.regions = true; + } else if ("VERBOSE".equalsIgnoreCase(name)) { + if (value != null) { + throw new RuntimeException("EXPLAIN option VERBOSE does not take a value"); + } + opts.mark(ExplainOpt.VERBOSE); + opts.verbose = true; + } else if ("FORMAT".equalsIgnoreCase(name)) { + if (value == null) { + throw new RuntimeException("EXPLAIN option FORMAT requires a value: TEXT or JSON"); + } + opts.mark(ExplainOpt.FORMAT); + if ("TEXT".equalsIgnoreCase(value)) { + opts.format = ExplainOptions.Format.TEXT; + } else if ("JSON".equalsIgnoreCase(value)) { + opts.format = ExplainOptions.Format.JSON; + } else { + throw new RuntimeException("Unknown EXPLAIN FORMAT: " + value); + } + } else { + throw new RuntimeException("Unknown EXPLAIN option: " + name); + } + } + protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow) throws RecognitionException { RecognitionException e = null; @@ -477,16 +541,25 @@ oneStatement returns [BindableStatement ret] finally{ contextStack.pop(); } explain_node returns [BindableStatement ret] - : EXPLAIN (w=WITH)? (r=REGIONS)? q=oneStatement +@init { ExplainOptsAcc opts = new ExplainOptsAcc(); } + : EXPLAIN + ( + (LPAREN explain_option[opts] (COMMA explain_option[opts])* RPAREN) + | (WITH REGIONS { opts.mark(ExplainOpt.REGIONS); opts.regions = true; }) + )? + q=oneStatement { - if ((w==null && r!=null) || (w!=null && r==null)) { - throw new RuntimeException("Valid usage: EXPLAIN {query} OR EXPLAIN WITH REGIONS {query}"); - } - ret = (w==null && r==null) ? factory.explain(q, ExplainType.DEFAULT) - : factory.explain(q, ExplainType.WITH_REGIONS); + ret = factory.explain(q, opts.build()); } ; +// A single EXPLAIN option. REGIONS is a global keyword token. The remaining option keywords +// (VERBOSE, FORMAT, TEXT, JSON) are not reserved and arrive as NAME tokens, validated in the action. +explain_option[ExplainOptsAcc opts] + : REGIONS { opts.mark(ExplainOpt.REGIONS); opts.regions = true; } + | k=NAME (v=NAME)? { parseExplainOption(opts, k.getText(), v == null ? null : v.getText()); } + ; + // Parse a create table statement. create_table_node returns [CreateTableStatement ret] : CREATE (im=IMMUTABLE)? TABLE (IF NOT ex=EXISTS)? t=from_table_name diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java index 3d1e9ffdcda..e73ad41dbe1 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java @@ -59,6 +59,7 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.index.PhoenixIndexCodec; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixResultSet; @@ -595,9 +596,11 @@ public MutationPlan compile(DeleteStatement delete, MutationState.ReturnResult r delete.getLimit(), null, delete.getBindCount(), false, false, Collections. emptyList(), delete.getUdfParseNodes()); select = StatementNormalizer.normalize(select, resolverToBe); - + // Pre-build a context so the early rewrite pass records top of plan breadcrumbs that are + // adopted by the DELETE data query plan's compilation context. + StatementContext rewriteContext = StatementContext.forRewrite(statement); SelectStatement transformedSelect = - SubqueryRewriter.transform(select, resolverToBe, connection); + SubqueryRewriter.transform(select, resolverToBe, connection, rewriteContext); boolean hasPreProcessing = transformedSelect != select; if (transformedSelect != select) { resolverToBe = FromCompiler.getResolverForQuery(transformedSelect, connection, false, @@ -624,7 +627,8 @@ public MutationPlan compile(DeleteStatement delete, MutationState.ReturnResult r QueryOptimizer optimizer = new QueryOptimizer(services); QueryCompiler compiler = new QueryCompiler(statement, select, resolverToBe, Collections. emptyList(), - parallelIteratorFactoryToBe, new SequenceManager(statement)); + parallelIteratorFactoryToBe, new SequenceManager(statement)) + .withRewriteContext(rewriteContext); final QueryPlan dataPlan = compiler.compile(); // TODO: the select clause should know that there's a sub query, but doesn't seem to currently queryPlans = Lists.newArrayList(!clientSideIndexes.isEmpty() @@ -667,7 +671,8 @@ Collections. emptyList(), parallelIteratorFactoryToBe) // from the data table, while the others will be for deleting rows from immutable indexes. List mutationPlans = Lists.newArrayListWithExpectedSize(queryPlans.size()); for (final QueryPlan plan : queryPlans) { - mutationPlans.add(new SingleRowDeleteMutationPlan(plan, connection, maxSize, maxSizeBytes)); + mutationPlans.add(new SingleRowDeleteMutationPlan(plan, connection, maxSize, maxSizeBytes, + delete.isReturningRow())); } return new MultiRowDeleteMutationPlan(dataPlan, mutationPlans); } else if (runOnServer) { @@ -700,7 +705,7 @@ Collections. emptyList(), parallelIteratorFactoryToBe) new AggregatePlan(context, select, dataPlan.getTableRef(), projector, null, null, OrderBy.EMPTY_ORDER_BY, null, GroupBy.EMPTY_GROUP_BY, null, dataPlan); return new ServerSelectDeleteMutationPlan(dataPlan, connection, aggPlan, projector, maxSize, - maxSizeBytes); + maxSizeBytes, delete.isReturningRow()); } else { final DeletingParallelIteratorFactory parallelIteratorFactory = parallelIteratorFactoryToBe; List adjustedProjectedColumns = @@ -745,7 +750,7 @@ public int getPosition() { } return new ClientSelectDeleteMutationPlan(targetTableRef, dataPlan, bestPlan, hasPreOrPostProcessing, parallelIteratorFactory, otherTableRefs, projectedTableRef, maxSize, - maxSizeBytes, connection); + maxSizeBytes, connection, delete.isReturningRow()); } } @@ -759,14 +764,16 @@ private class SingleRowDeleteMutationPlan implements MutationPlan { private final int maxSize; private final StatementContext context; private final long maxSizeBytes; + private final boolean returningRow; public SingleRowDeleteMutationPlan(QueryPlan dataPlan, PhoenixConnection connection, - int maxSize, long maxSizeBytes) { + int maxSize, long maxSizeBytes, boolean returningRow) { this.dataPlan = dataPlan; this.connection = connection; this.maxSize = maxSize; this.context = dataPlan.getContext(); this.maxSizeBytes = maxSizeBytes; + this.returningRow = returningRow; } @Override @@ -793,7 +800,19 @@ public MutationState execute() throws SQLException { @Override public ExplainPlan getExplainPlan() throws SQLException { - return new ExplainPlan(Collections.singletonList("DELETE SINGLE ROW")); + ExplainPlanAttributesBuilder builder = + new ExplainPlanAttributesBuilder().setAbstractExplainPlan("DELETE SINGLE ROW"); + builder.setReturningRow(returningRow); + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); + } + List planSteps = Lists.newArrayListWithExpectedSize(2); + planSteps.add("DELETE SINGLE ROW"); + if (returningRow) { + planSteps.add(" RETURNING *"); + } + return new ExplainPlan(planSteps, builder.build()); } @Override @@ -852,9 +871,11 @@ public class ServerSelectDeleteMutationPlan implements MutationPlan { private final RowProjector projector; private final int maxSize; private final long maxSizeBytes; + private final boolean returningRow; public ServerSelectDeleteMutationPlan(QueryPlan dataPlan, PhoenixConnection connection, - QueryPlan aggPlan, RowProjector projector, int maxSize, long maxSizeBytes) { + QueryPlan aggPlan, RowProjector projector, int maxSize, long maxSizeBytes, + boolean returningRow) { this.context = dataPlan.getContext(); this.dataPlan = dataPlan; this.connection = connection; @@ -862,6 +883,7 @@ public ServerSelectDeleteMutationPlan(QueryPlan dataPlan, PhoenixConnection conn this.projector = projector; this.maxSize = maxSize; this.maxSizeBytes = maxSizeBytes; + this.returningRow = returningRow; } @Override @@ -970,12 +992,24 @@ public ExplainPlan getExplainPlan() throws SQLException { ExplainPlan explainPlan = aggPlan.getExplainPlan(); List queryPlanSteps = explainPlan.getPlanSteps(); ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - List planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 1); + List planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 2); ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); newBuilder.setAbstractExplainPlan("DELETE ROWS SERVER SELECT"); + newBuilder.setReturningRow(returningRow); planSteps.add("DELETE ROWS SERVER SELECT"); + if (returningRow) { + planSteps.add(" RETURNING *"); + } planSteps.addAll(queryPlanSteps); + // Surface the row-identity projection the scan actually reads so VERBOSE explain describes + // the delete rather than the count. + ExplainTable.overrideMutationProject(planSteps, explainPlanAttributes, newBuilder, + dataPlan.getProjector()); + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); + } return new ExplainPlan(planSteps, newBuilder.build()); } @@ -1016,11 +1050,13 @@ public class ClientSelectDeleteMutationPlan implements MutationPlan { private final int maxSize; private final long maxSizeBytes; private final PhoenixConnection connection; + private final boolean returningRow; public ClientSelectDeleteMutationPlan(TableRef targetTableRef, QueryPlan dataPlan, QueryPlan bestPlan, boolean hasPreOrPostProcessing, DeletingParallelIteratorFactory parallelIteratorFactory, List otherTableRefs, - TableRef projectedTableRef, int maxSize, long maxSizeBytes, PhoenixConnection connection) { + TableRef projectedTableRef, int maxSize, long maxSizeBytes, PhoenixConnection connection, + boolean returningRow) { this.context = bestPlan.getContext(); this.targetTableRef = targetTableRef; this.dataPlan = dataPlan; @@ -1032,6 +1068,7 @@ public ClientSelectDeleteMutationPlan(TableRef targetTableRef, QueryPlan dataPla this.maxSize = maxSize; this.maxSizeBytes = maxSizeBytes; this.connection = connection; + this.returningRow = returningRow; } @Override @@ -1105,12 +1142,20 @@ public ExplainPlan getExplainPlan() throws SQLException { ExplainPlan explainPlan = bestPlan.getExplainPlan(); List queryPlanSteps = explainPlan.getPlanSteps(); ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - List planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 1); + List planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 2); ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); newBuilder.setAbstractExplainPlan("DELETE ROWS CLIENT SELECT"); + newBuilder.setReturningRow(returningRow); planSteps.add("DELETE ROWS CLIENT SELECT"); + if (returningRow) { + planSteps.add(" RETURNING *"); + } planSteps.addAll(queryPlanSteps); + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); + } return new ExplainPlan(planSteps, newBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainJsonRenderer.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainJsonRenderer.java new file mode 100644 index 00000000000..fed80dc9950 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainJsonRenderer.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compile; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.util.DefaultIndenter; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import java.sql.SQLException; + +/** + * Serializes an {@link ExplainPlanAttributes} tree to a JSON document for the + * {@code EXPLAIN (FORMAT JSON) } statement. + *

+ * The output is pretty-printed with two space indentation for both objects and arrays. + *

+ * The JSON layout tracks the Java field names and structure of {@link ExplainPlanAttributes}. It is + * deliberately not a stable contract and carries no version field. It is an opt-in view onto an + * internal structure, useful for tooling and assertions. + *

+ * This class intentionally does not reuse the shared {@link org.apache.phoenix.util.JacksonUtil} + * mapper so that the general-purpose mapper configuration can change without affecting the EXPLAIN + * JSON contract. + */ +public final class ExplainJsonRenderer { + + private static final ObjectWriter WRITER = buildWriter(); + + private static ObjectWriter buildWriter() { + ObjectMapper mapper = new ObjectMapper(); + // Emit every field, with an explicit null for any unset value, so the JSON view is a faithful + // projection of the attributes tree. + mapper.setSerializationInclusion(JsonInclude.Include.ALWAYS); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + // Jackson's stock DefaultPrettyPrinter indents object fields but uses a single space + // FixedSpaceIndenter for array elements. Set the same two space indenter on both so nested + // objects and array elements each start on their own indented line. + DefaultIndenter indenter = new DefaultIndenter(" ", "\n"); + DefaultPrettyPrinter printer = + new DefaultPrettyPrinter().withObjectIndenter(indenter).withArrayIndenter(indenter); + return mapper.writer(printer); + } + + private ExplainJsonRenderer() { + } + + /** + * Serialize the given attributes to a pretty-printed JSON document. + * @param attributes the plan attributes to serialize + * @return the JSON document + * @throws SQLException if serialization fails + */ + public static String render(ExplainPlanAttributes attributes) throws SQLException { + try { + return WRITER.writeValueAsString(attributes); + } catch (JsonProcessingException e) { + throw new SQLException("Failed to serialize EXPLAIN attributes as JSON", e); + } + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index 07f00f2b672..e233931f71d 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -17,12 +17,21 @@ */ package org.apache.phoenix.compile; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Consistency; +import org.apache.phoenix.optimize.RejectedIndexEntry; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; +import org.apache.phoenix.parse.UpsertStatement.OnDuplicateKeyType; import org.apache.phoenix.schema.PColumn; /** @@ -30,191 +39,261 @@ * against. This also makes attribute retrieval easier as an API rather than retrieving list of * Strings containing entire plan. */ +@JsonPropertyOrder({ "tenantId", "viewName", "viewBaseName", "cdcScopes", "txnProvider", "rewrites", + "estimatedRows", "estimatedSizeInBytes", "estimateInfoTs", "abstractExplainPlan", + "onDuplicateKeyAction", "serverUpdateSet", "returningRow", "hint", "explainScanType", + "consistency", "tableName", "keyRanges", "indexName", "indexKind", "indexRule", "functionalMatch", + "indexRejected", "saltBuckets", "regionsPlanned", "scanTimeRangeMin", "scanTimeRangeMax", + "splitsChunk", "useRoundRobinIterator", "samplingRate", "hexStringRVCOffset", + "iteratorTypeAndScanSize", "scanEstimatedRows", "scanEstimatedSizeInBytes", "serverWhereFilter", + "serverDistinctFilter", "serverMergeColumns", "serverParsedProjections", "serverProject", + "serverFilters", "ignoredHints", "serverFirstKeyOnlyProjection", + "serverEmptyColumnOnlyProjection", "serverAggregate", "serverGroupByLimit", "serverSortedBy", + "serverOffset", "serverRowLimit", "clientFilterBy", "clientFilters", "clientAggregate", + "clientDistinctFilter", "clientAfterAggregate", "clientSortAlgo", "clientSortedBy", + "clientOffset", "clientRowLimit", "clientSequenceCount", "clientCursorName", "clientSteps", + "lhsJoinQueryExplainPlan", "rhsJoinQueryExplainPlan", "subPlans", "dynamicServerFilter", + "afterJoinFilter", "joinScannerLimit", "sortMergeSkipMerge", "regionLocations", + "regionLocationsTotalSize", "numRegionLocationLookups" }) public class ExplainPlanAttributes { - private final String abstractExplainPlan; - private final Integer splitsChunk; + // Top-of-plan disclosures (populated only on the root plan) + private final String tenantId; + private final String viewName; + private final String viewBaseName; + private final String cdcScopes; + private final String txnProvider; + private final List rewrites; + // Plan-total estimates. private final Long estimatedRows; private final Long estimatedSizeInBytes; - private final String iteratorTypeAndScanSize; - private final Double samplingRate; - private final boolean useRoundRobinIterator; - private final String hexStringRVCOffset; - private final Consistency consistency; + private final Long estimateInfoTs; + + // Plan identity and scan-level metadata + private final String abstractExplainPlan; + // Mutation-operator detail (populated only on mutation plans). + private final OnDuplicateKeyType onDuplicateKeyAction; + private final List serverUpdateSet; + private final boolean returningRow; private final Hint hint; - private final String serverSortedBy; private final String explainScanType; + private final Consistency consistency; private final String tableName; private final String keyRanges; + private final String indexName; + private final String indexKind; + private final String indexRule; + private final String functionalMatch; + private final List indexRejected; + private final Integer saltBuckets; + private final Integer regionsPlanned; private final Long scanTimeRangeMin; private final Long scanTimeRangeMax; + private final Integer splitsChunk; + private final boolean useRoundRobinIterator; + private final Double samplingRate; + private final String hexStringRVCOffset; + private final String iteratorTypeAndScanSize; + // Per-scan estimates (populated on each plan level from stats). + private final Long scanEstimatedRows; + private final Long scanEstimatedSizeInBytes; + + // Server-side operations private final String serverWhereFilter; private final String serverDistinctFilter; + private final Set serverMergeColumns; + private final Map> serverParsedProjections; + private final List serverProject; + private final List serverFilters; + private final Map ignoredHints; + private final boolean serverFirstKeyOnlyProjection; + private final boolean serverEmptyColumnOnlyProjection; + private final String serverAggregate; + private final Integer serverGroupByLimit; + private final String serverSortedBy; private final Integer serverOffset; private final Long serverRowLimit; - private final boolean serverArrayElementProjection; - private final String serverAggregate; + + // Client-side operations private final String clientFilterBy; + private final List clientFilters; private final String clientAggregate; - private final String clientSortedBy; - private final String clientAfterAggregate; private final String clientDistinctFilter; + private final String clientAfterAggregate; + private final String clientSortAlgo; + private final String clientSortedBy; private final Integer clientOffset; private final Integer clientRowLimit; private final Integer clientSequenceCount; private final String clientCursorName; - private final String clientSortAlgo; - // This object represents PlanAttributes object for rhs query - // to be used only by Join queries. In case of Join query, lhs plan is - // represented by 'this' object and rhs plan is represented by - // 'rhsJoinQueryExplainPlan' object (which in turn should - // have null rhsJoinQueryExplainPlan) - // For non-Join queries related Plans, rhsJoinQueryExplainPlan will always - // be null + // Ordered client-side pipeline (CLIENT* lines in emission order). + private final List clientSteps; + + // Join / sub-plan + private final ExplainPlanAttributes lhsJoinQueryExplainPlan; private final ExplainPlanAttributes rhsJoinQueryExplainPlan; - private final Set serverMergeColumns; + private final List subPlans; + private final String dynamicServerFilter; + private final String afterJoinFilter; + private final Long joinScannerLimit; + private final boolean sortMergeSkipMerge; + + // Region-location metadata private final List regionLocations; + // Total number of distinct region locations before any trimming. Null when not populated. + private final Integer regionLocationsTotalSize; private final int numRegionLocationLookups; - private static final ExplainPlanAttributes EXPLAIN_PLAN_INSTANCE = new ExplainPlanAttributes(); - - private ExplainPlanAttributes() { - this.abstractExplainPlan = null; - this.splitsChunk = null; - this.estimatedRows = null; - this.estimatedSizeInBytes = null; - this.iteratorTypeAndScanSize = null; - this.samplingRate = null; - this.useRoundRobinIterator = false; - this.hexStringRVCOffset = null; - this.consistency = null; - this.hint = null; - this.serverSortedBy = null; - this.explainScanType = null; - this.tableName = null; - this.keyRanges = null; - this.scanTimeRangeMin = null; - this.scanTimeRangeMax = null; - this.serverWhereFilter = null; - this.serverDistinctFilter = null; - this.serverOffset = null; - this.serverRowLimit = null; - this.serverArrayElementProjection = false; - this.serverAggregate = null; - this.clientFilterBy = null; - this.clientAggregate = null; - this.clientSortedBy = null; - this.clientAfterAggregate = null; - this.clientDistinctFilter = null; - this.clientOffset = null; - this.clientRowLimit = null; - this.clientSequenceCount = null; - this.clientCursorName = null; - this.clientSortAlgo = null; - this.rhsJoinQueryExplainPlan = null; - this.serverMergeColumns = null; - this.regionLocations = null; - this.numRegionLocationLookups = 0; - } - - public ExplainPlanAttributes(String abstractExplainPlan, Integer splitsChunk, Long estimatedRows, - Long estimatedSizeInBytes, String iteratorTypeAndScanSize, Double samplingRate, - boolean useRoundRobinIterator, String hexStringRVCOffset, Consistency consistency, Hint hint, - String serverSortedBy, String explainScanType, String tableName, String keyRanges, - Long scanTimeRangeMin, Long scanTimeRangeMax, String serverWhereFilter, - String serverDistinctFilter, Integer serverOffset, Long serverRowLimit, - boolean serverArrayElementProjection, String serverAggregate, String clientFilterBy, - String clientAggregate, String clientSortedBy, String clientAfterAggregate, - String clientDistinctFilter, Integer clientOffset, Integer clientRowLimit, - Integer clientSequenceCount, String clientCursorName, String clientSortAlgo, - ExplainPlanAttributes rhsJoinQueryExplainPlan, Set serverMergeColumns, - List regionLocations, int numRegionLocationLookups) { - this.abstractExplainPlan = abstractExplainPlan; - this.splitsChunk = splitsChunk; - this.estimatedRows = estimatedRows; - this.estimatedSizeInBytes = estimatedSizeInBytes; - this.iteratorTypeAndScanSize = iteratorTypeAndScanSize; - this.samplingRate = samplingRate; - this.useRoundRobinIterator = useRoundRobinIterator; - this.hexStringRVCOffset = hexStringRVCOffset; - this.consistency = consistency; - this.hint = hint; - this.serverSortedBy = serverSortedBy; - this.explainScanType = explainScanType; - this.tableName = tableName; - this.keyRanges = keyRanges; - this.scanTimeRangeMin = scanTimeRangeMin; - this.scanTimeRangeMax = scanTimeRangeMax; - this.serverWhereFilter = serverWhereFilter; - this.serverDistinctFilter = serverDistinctFilter; - this.serverOffset = serverOffset; - this.serverRowLimit = serverRowLimit; - this.serverArrayElementProjection = serverArrayElementProjection; - this.serverAggregate = serverAggregate; - this.clientFilterBy = clientFilterBy; - this.clientAggregate = clientAggregate; - this.clientSortedBy = clientSortedBy; - this.clientAfterAggregate = clientAfterAggregate; - this.clientDistinctFilter = clientDistinctFilter; - this.clientOffset = clientOffset; - this.clientRowLimit = clientRowLimit; - this.clientSequenceCount = clientSequenceCount; - this.clientCursorName = clientCursorName; - this.clientSortAlgo = clientSortAlgo; - this.rhsJoinQueryExplainPlan = rhsJoinQueryExplainPlan; - this.serverMergeColumns = serverMergeColumns; - this.regionLocations = regionLocations; - this.numRegionLocationLookups = numRegionLocationLookups; + private static final ExplainPlanAttributes EXPLAIN_PLAN_INSTANCE = + new ExplainPlanAttributesBuilder().build(); + + private ExplainPlanAttributes(ExplainPlanAttributesBuilder b) { + this.tenantId = b.tenantId; + this.viewName = b.viewName; + this.viewBaseName = b.viewBaseName; + this.cdcScopes = b.cdcScopes; + this.txnProvider = b.txnProvider; + this.rewrites = (b.rewrites == null || b.rewrites.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.rewrites)); + this.estimatedRows = b.estimatedRows; + this.estimatedSizeInBytes = b.estimatedSizeInBytes; + this.estimateInfoTs = b.estimateInfoTs; + this.abstractExplainPlan = b.abstractExplainPlan; + this.onDuplicateKeyAction = b.onDuplicateKeyAction; + this.serverUpdateSet = (b.serverUpdateSet == null || b.serverUpdateSet.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.serverUpdateSet)); + this.returningRow = b.returningRow; + this.hint = b.hint; + this.explainScanType = b.explainScanType; + this.consistency = b.consistency; + this.tableName = b.tableName; + this.keyRanges = b.keyRanges; + this.indexName = b.indexName; + this.indexKind = b.indexKind; + this.indexRule = b.indexRule; + this.functionalMatch = b.functionalMatch; + this.indexRejected = (b.indexRejected == null || b.indexRejected.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.indexRejected)); + this.saltBuckets = b.saltBuckets; + this.regionsPlanned = b.regionsPlanned; + this.scanTimeRangeMin = b.scanTimeRangeMin; + this.scanTimeRangeMax = b.scanTimeRangeMax; + this.splitsChunk = b.splitsChunk; + this.useRoundRobinIterator = b.useRoundRobinIterator; + this.samplingRate = b.samplingRate; + this.hexStringRVCOffset = b.hexStringRVCOffset; + this.iteratorTypeAndScanSize = b.iteratorTypeAndScanSize; + this.scanEstimatedRows = b.scanEstimatedRows; + this.scanEstimatedSizeInBytes = b.scanEstimatedSizeInBytes; + this.serverWhereFilter = b.serverWhereFilter; + this.serverDistinctFilter = b.serverDistinctFilter; + this.serverMergeColumns = (b.serverMergeColumns == null || b.serverMergeColumns.isEmpty()) + ? null + : Collections.unmodifiableSet(new LinkedHashSet<>(b.serverMergeColumns)); + this.serverParsedProjections = copyServerParsedProjections(b.serverParsedProjections); + this.serverProject = (b.serverProject == null || b.serverProject.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.serverProject)); + this.serverFilters = (b.serverFilters == null || b.serverFilters.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.serverFilters)); + this.ignoredHints = (b.ignoredHints == null || b.ignoredHints.isEmpty()) + ? null + : Collections.unmodifiableMap(new LinkedHashMap<>(b.ignoredHints)); + this.serverFirstKeyOnlyProjection = b.serverFirstKeyOnlyProjection; + this.serverEmptyColumnOnlyProjection = b.serverEmptyColumnOnlyProjection; + this.serverAggregate = b.serverAggregate; + this.serverGroupByLimit = b.serverGroupByLimit; + this.serverSortedBy = b.serverSortedBy; + this.serverOffset = b.serverOffset; + this.serverRowLimit = b.serverRowLimit; + this.clientFilterBy = b.clientFilterBy; + this.clientFilters = (b.clientFilters == null || b.clientFilters.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.clientFilters)); + this.clientAggregate = b.clientAggregate; + this.clientDistinctFilter = b.clientDistinctFilter; + this.clientAfterAggregate = b.clientAfterAggregate; + this.clientSortAlgo = b.clientSortAlgo; + this.clientSortedBy = b.clientSortedBy; + this.clientOffset = b.clientOffset; + this.clientRowLimit = b.clientRowLimit; + this.clientSequenceCount = b.clientSequenceCount; + this.clientCursorName = b.clientCursorName; + this.clientSteps = (b.clientSteps == null || b.clientSteps.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.clientSteps)); + this.lhsJoinQueryExplainPlan = b.lhsJoinQueryExplainPlan; + this.rhsJoinQueryExplainPlan = b.rhsJoinQueryExplainPlan; + this.subPlans = (b.subPlans == null || b.subPlans.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.subPlans)); + this.dynamicServerFilter = b.dynamicServerFilter; + this.afterJoinFilter = b.afterJoinFilter; + this.joinScannerLimit = b.joinScannerLimit; + this.sortMergeSkipMerge = b.sortMergeSkipMerge; + this.regionLocations = (b.regionLocations == null || b.regionLocations.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(b.regionLocations)); + this.regionLocationsTotalSize = b.regionLocationsTotalSize; + this.numRegionLocationLookups = b.numRegionLocationLookups; } - public String getAbstractExplainPlan() { - return abstractExplainPlan; + public String getTenantId() { + return tenantId; } - public Integer getSplitsChunk() { - return splitsChunk; + public String getViewName() { + return viewName; } - public Long getEstimatedRows() { - return estimatedRows; + public String getViewBaseName() { + return viewBaseName; } - public Long getEstimatedSizeInBytes() { - return estimatedSizeInBytes; + public String getCdcScopes() { + return cdcScopes; } - public String getIteratorTypeAndScanSize() { - return iteratorTypeAndScanSize; + public String getTxnProvider() { + return txnProvider; } - public Double getSamplingRate() { - return samplingRate; + public List getRewrites() { + return rewrites; } - public boolean isUseRoundRobinIterator() { - return useRoundRobinIterator; + public String getAbstractExplainPlan() { + return abstractExplainPlan; } - public String getHexStringRVCOffset() { - return hexStringRVCOffset; + public OnDuplicateKeyType getOnDuplicateKeyAction() { + return onDuplicateKeyAction; } - public Consistency getConsistency() { - return consistency; + public List getServerUpdateSet() { + return serverUpdateSet; } - public Hint getHint() { - return hint; + public boolean isReturningRow() { + return returningRow; } - public String getServerSortedBy() { - return serverSortedBy; + public Hint getHint() { + return hint; } public String getExplainScanType() { return explainScanType; } + public Consistency getConsistency() { + return consistency; + } + public String getTableName() { return tableName; } @@ -223,6 +302,34 @@ public String getKeyRanges() { return keyRanges; } + public String getIndexName() { + return indexName; + } + + public String getIndexKind() { + return indexKind; + } + + public String getIndexRule() { + return indexRule; + } + + public String getFunctionalMatch() { + return functionalMatch; + } + + public List getIndexRejected() { + return indexRejected; + } + + public Integer getSaltBuckets() { + return saltBuckets; + } + + public Integer getRegionsPlanned() { + return regionsPlanned; + } + public Long getScanTimeRangeMin() { return scanTimeRangeMin; } @@ -231,6 +338,46 @@ public Long getScanTimeRangeMax() { return scanTimeRangeMax; } + public Integer getSplitsChunk() { + return splitsChunk; + } + + public boolean isUseRoundRobinIterator() { + return useRoundRobinIterator; + } + + public Double getSamplingRate() { + return samplingRate; + } + + public String getHexStringRVCOffset() { + return hexStringRVCOffset; + } + + public String getIteratorTypeAndScanSize() { + return iteratorTypeAndScanSize; + } + + public Long getScanEstimatedRows() { + return scanEstimatedRows; + } + + public Long getScanEstimatedSizeInBytes() { + return scanEstimatedSizeInBytes; + } + + public Long getEstimatedRows() { + return estimatedRows; + } + + public Long getEstimatedSizeInBytes() { + return estimatedSizeInBytes; + } + + public Long getEstimateInfoTs() { + return estimateInfoTs; + } + public String getServerWhereFilter() { return serverWhereFilter; } @@ -239,40 +386,93 @@ public String getServerDistinctFilter() { return serverDistinctFilter; } - public Integer getServerOffset() { - return serverOffset; + @JsonSerialize(using = ServerMergeColumnsSerializer.class) + public Set getServerMergeColumns() { + return serverMergeColumns; } - public Long getServerRowLimit() { - return serverRowLimit; + public Map> getServerParsedProjections() { + return serverParsedProjections; } - public boolean isServerArrayElementProjection() { - return serverArrayElementProjection; + private static Map> + copyServerParsedProjections(Map> source) { + if (source == null || source.isEmpty()) { + return null; + } + Map> copy = new LinkedHashMap<>(); + for (Map.Entry> entry : source.entrySet()) { + copy.put(entry.getKey(), Collections.unmodifiableList(new ArrayList<>(entry.getValue()))); + } + return Collections.unmodifiableMap(copy); + } + + public List getServerProject() { + return serverProject; + } + + public List getServerFilters() { + return serverFilters; + } + + public Map getIgnoredHints() { + return ignoredHints; + } + + public boolean isServerFirstKeyOnlyProjection() { + return serverFirstKeyOnlyProjection; + } + + public boolean isServerEmptyColumnOnlyProjection() { + return serverEmptyColumnOnlyProjection; } public String getServerAggregate() { return serverAggregate; } + public Integer getServerGroupByLimit() { + return serverGroupByLimit; + } + + public String getServerSortedBy() { + return serverSortedBy; + } + + public Integer getServerOffset() { + return serverOffset; + } + + public Long getServerRowLimit() { + return serverRowLimit; + } + public String getClientFilterBy() { return clientFilterBy; } + public List getClientFilters() { + return clientFilters; + } + public String getClientAggregate() { return clientAggregate; } - public String getClientSortedBy() { - return clientSortedBy; + public String getClientDistinctFilter() { + return clientDistinctFilter; } public String getClientAfterAggregate() { return clientAfterAggregate; } - public String getClientDistinctFilter() { - return clientDistinctFilter; + public String getClientSortAlgo() { + return clientSortAlgo; + } + + public String getClientSortedBy() { + return clientSortedBy; } public Integer getClientOffset() { @@ -291,22 +491,47 @@ public String getClientCursorName() { return clientCursorName; } - public String getClientSortAlgo() { - return clientSortAlgo; + public List getClientSteps() { + return clientSteps; + } + + public ExplainPlanAttributes getLhsJoinQueryExplainPlan() { + return lhsJoinQueryExplainPlan; } public ExplainPlanAttributes getRhsJoinQueryExplainPlan() { return rhsJoinQueryExplainPlan; } - public Set getServerMergeColumns() { - return serverMergeColumns; + public List getSubPlans() { + return subPlans; + } + + public String getDynamicServerFilter() { + return dynamicServerFilter; } + public String getAfterJoinFilter() { + return afterJoinFilter; + } + + public Long getJoinScannerLimit() { + return joinScannerLimit; + } + + public boolean isSortMergeSkipMerge() { + return sortMergeSkipMerge; + } + + @JsonSerialize(using = RegionLocationsListSerializer.class) public List getRegionLocations() { return regionLocations; } + public Integer getRegionLocationsTotalSize() { + return regionLocationsTotalSize; + } + public int getNumRegionLocationLookups() { return numRegionLocationLookups; } @@ -315,42 +540,129 @@ public static ExplainPlanAttributes getDefaultExplainPlan() { return EXPLAIN_PLAN_INSTANCE; } + /** A single VERBOSE-mode filter predicate. */ + @JsonPropertyOrder({ "expr", "origin", "pathTestSubtag" }) + public static class ExplainFilter { + private final String expr; + private final List origin; + private final String pathTestSubtag; + + public ExplainFilter(String expr, List origin, String pathTestSubtag) { + this.expr = expr; + this.origin = (origin == null || origin.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(origin)); + this.pathTestSubtag = pathTestSubtag; + } + + public String getExpr() { + return expr; + } + + public List getOrigin() { + return origin; + } + + public String getPathTestSubtag() { + return pathTestSubtag; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ExplainFilter)) { + return false; + } + ExplainFilter that = (ExplainFilter) o; + return java.util.Objects.equals(expr, that.expr) + && java.util.Objects.equals(origin, that.origin) + && java.util.Objects.equals(pathTestSubtag, that.pathTestSubtag); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(expr, origin, pathTestSubtag); + } + + @Override + public String toString() { + return "ExplainFilter{expr=" + expr + ", origin=" + origin + ", pathTestSubtag=" + + pathTestSubtag + "}"; + } + } + public static class ExplainPlanAttributesBuilder { - private String abstractExplainPlan; - private Integer splitsChunk; + private String tenantId; + private String viewName; + private String viewBaseName; + private String cdcScopes; + private String txnProvider; + private List rewrites; private Long estimatedRows; private Long estimatedSizeInBytes; - private String iteratorTypeAndScanSize; - private Double samplingRate; - private boolean useRoundRobinIterator; - private String hexStringRVCOffset; - private Consistency consistency; + private Long estimateInfoTs; + private String abstractExplainPlan; + private OnDuplicateKeyType onDuplicateKeyAction; + private List serverUpdateSet; + private boolean returningRow; private HintNode.Hint hint; - private String serverSortedBy; private String explainScanType; + private Consistency consistency; private String tableName; private String keyRanges; + private String indexName; + private String indexKind; + private String indexRule; + private String functionalMatch; + private List indexRejected; + private Integer saltBuckets; + private Integer regionsPlanned; private Long scanTimeRangeMin; private Long scanTimeRangeMax; + private Integer splitsChunk; + private boolean useRoundRobinIterator; + private Double samplingRate; + private String hexStringRVCOffset; + private String iteratorTypeAndScanSize; + private Long scanEstimatedRows; + private Long scanEstimatedSizeInBytes; private String serverWhereFilter; private String serverDistinctFilter; + private Set serverMergeColumns; + private Map> serverParsedProjections; + private List serverProject; + private List serverFilters; + private Map ignoredHints; + private boolean serverFirstKeyOnlyProjection; + private boolean serverEmptyColumnOnlyProjection; + private String serverAggregate; + private Integer serverGroupByLimit; + private String serverSortedBy; private Integer serverOffset; private Long serverRowLimit; - private boolean serverArrayElementProjection; - private String serverAggregate; private String clientFilterBy; + private List clientFilters; private String clientAggregate; - private String clientSortedBy; - private String clientAfterAggregate; private String clientDistinctFilter; + private String clientAfterAggregate; + private String clientSortAlgo; + private String clientSortedBy; private Integer clientOffset; private Integer clientRowLimit; private Integer clientSequenceCount; private String clientCursorName; - private String clientSortAlgo; + private List clientSteps; + private ExplainPlanAttributes lhsJoinQueryExplainPlan; private ExplainPlanAttributes rhsJoinQueryExplainPlan; - private Set serverMergeColumns; + private List subPlans; + private String dynamicServerFilter; + private String afterJoinFilter; + private Long joinScannerLimit; + private boolean sortMergeSkipMerge; private List regionLocations; + private Integer regionLocationsTotalSize; private int numRegionLocationLookups; public ExplainPlanAttributesBuilder() { @@ -358,51 +670,126 @@ public ExplainPlanAttributesBuilder() { } public ExplainPlanAttributesBuilder(ExplainPlanAttributes explainPlanAttributes) { - this.abstractExplainPlan = explainPlanAttributes.getAbstractExplainPlan(); - this.splitsChunk = explainPlanAttributes.getSplitsChunk(); + this.tenantId = explainPlanAttributes.getTenantId(); + this.viewName = explainPlanAttributes.getViewName(); + this.viewBaseName = explainPlanAttributes.getViewBaseName(); + this.cdcScopes = explainPlanAttributes.getCdcScopes(); + this.txnProvider = explainPlanAttributes.getTxnProvider(); + List srcRewrites = explainPlanAttributes.getRewrites(); + this.rewrites = srcRewrites == null ? null : new ArrayList<>(srcRewrites); this.estimatedRows = explainPlanAttributes.getEstimatedRows(); this.estimatedSizeInBytes = explainPlanAttributes.getEstimatedSizeInBytes(); - this.iteratorTypeAndScanSize = explainPlanAttributes.getIteratorTypeAndScanSize(); - this.samplingRate = explainPlanAttributes.getSamplingRate(); - this.useRoundRobinIterator = explainPlanAttributes.isUseRoundRobinIterator(); - this.hexStringRVCOffset = explainPlanAttributes.getHexStringRVCOffset(); - this.consistency = explainPlanAttributes.getConsistency(); + this.estimateInfoTs = explainPlanAttributes.getEstimateInfoTs(); + this.abstractExplainPlan = explainPlanAttributes.getAbstractExplainPlan(); + this.onDuplicateKeyAction = explainPlanAttributes.getOnDuplicateKeyAction(); + List srcServerUpdateSet = explainPlanAttributes.getServerUpdateSet(); + this.serverUpdateSet = + srcServerUpdateSet == null ? null : new ArrayList<>(srcServerUpdateSet); + this.returningRow = explainPlanAttributes.isReturningRow(); this.hint = explainPlanAttributes.getHint(); - this.serverSortedBy = explainPlanAttributes.getServerSortedBy(); this.explainScanType = explainPlanAttributes.getExplainScanType(); + this.consistency = explainPlanAttributes.getConsistency(); this.tableName = explainPlanAttributes.getTableName(); this.keyRanges = explainPlanAttributes.getKeyRanges(); + this.indexName = explainPlanAttributes.getIndexName(); + this.indexKind = explainPlanAttributes.getIndexKind(); + this.indexRule = explainPlanAttributes.getIndexRule(); + this.functionalMatch = explainPlanAttributes.getFunctionalMatch(); + List srcIndexRejected = explainPlanAttributes.getIndexRejected(); + this.indexRejected = srcIndexRejected == null ? null : new ArrayList<>(srcIndexRejected); + this.saltBuckets = explainPlanAttributes.getSaltBuckets(); + this.regionsPlanned = explainPlanAttributes.getRegionsPlanned(); this.scanTimeRangeMin = explainPlanAttributes.getScanTimeRangeMin(); this.scanTimeRangeMax = explainPlanAttributes.getScanTimeRangeMax(); + this.splitsChunk = explainPlanAttributes.getSplitsChunk(); + this.useRoundRobinIterator = explainPlanAttributes.isUseRoundRobinIterator(); + this.samplingRate = explainPlanAttributes.getSamplingRate(); + this.hexStringRVCOffset = explainPlanAttributes.getHexStringRVCOffset(); + this.iteratorTypeAndScanSize = explainPlanAttributes.getIteratorTypeAndScanSize(); + this.scanEstimatedRows = explainPlanAttributes.getScanEstimatedRows(); + this.scanEstimatedSizeInBytes = explainPlanAttributes.getScanEstimatedSizeInBytes(); this.serverWhereFilter = explainPlanAttributes.getServerWhereFilter(); this.serverDistinctFilter = explainPlanAttributes.getServerDistinctFilter(); + this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns(); + Map> srcServerParsedProjections = + explainPlanAttributes.getServerParsedProjections(); + this.serverParsedProjections = + srcServerParsedProjections == null ? null : new LinkedHashMap<>(srcServerParsedProjections); + List srcServerProject = explainPlanAttributes.getServerProject(); + this.serverProject = srcServerProject == null ? null : new ArrayList<>(srcServerProject); + List srcServerFilters = explainPlanAttributes.getServerFilters(); + this.serverFilters = srcServerFilters == null ? null : new ArrayList<>(srcServerFilters); + Map srcIgnoredHints = explainPlanAttributes.getIgnoredHints(); + this.ignoredHints = srcIgnoredHints == null ? null : new LinkedHashMap<>(srcIgnoredHints); + this.serverFirstKeyOnlyProjection = explainPlanAttributes.isServerFirstKeyOnlyProjection(); + this.serverEmptyColumnOnlyProjection = + explainPlanAttributes.isServerEmptyColumnOnlyProjection(); + this.serverAggregate = explainPlanAttributes.getServerAggregate(); + this.serverGroupByLimit = explainPlanAttributes.getServerGroupByLimit(); + this.serverSortedBy = explainPlanAttributes.getServerSortedBy(); this.serverOffset = explainPlanAttributes.getServerOffset(); this.serverRowLimit = explainPlanAttributes.getServerRowLimit(); - this.serverArrayElementProjection = explainPlanAttributes.isServerArrayElementProjection(); - this.serverAggregate = explainPlanAttributes.getServerAggregate(); this.clientFilterBy = explainPlanAttributes.getClientFilterBy(); + List srcClientFilters = explainPlanAttributes.getClientFilters(); + this.clientFilters = srcClientFilters == null ? null : new ArrayList<>(srcClientFilters); this.clientAggregate = explainPlanAttributes.getClientAggregate(); - this.clientSortedBy = explainPlanAttributes.getClientSortedBy(); - this.clientAfterAggregate = explainPlanAttributes.getClientAfterAggregate(); this.clientDistinctFilter = explainPlanAttributes.getClientDistinctFilter(); + this.clientAfterAggregate = explainPlanAttributes.getClientAfterAggregate(); + this.clientSortAlgo = explainPlanAttributes.getClientSortAlgo(); + this.clientSortedBy = explainPlanAttributes.getClientSortedBy(); this.clientOffset = explainPlanAttributes.getClientOffset(); this.clientRowLimit = explainPlanAttributes.getClientRowLimit(); this.clientSequenceCount = explainPlanAttributes.getClientSequenceCount(); this.clientCursorName = explainPlanAttributes.getClientCursorName(); - this.clientSortAlgo = explainPlanAttributes.getClientSortAlgo(); + List srcClientSteps = explainPlanAttributes.getClientSteps(); + this.clientSteps = srcClientSteps == null ? null : new ArrayList<>(srcClientSteps); + this.lhsJoinQueryExplainPlan = explainPlanAttributes.getLhsJoinQueryExplainPlan(); this.rhsJoinQueryExplainPlan = explainPlanAttributes.getRhsJoinQueryExplainPlan(); - this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns(); + this.subPlans = explainPlanAttributes.getSubPlans(); + this.dynamicServerFilter = explainPlanAttributes.getDynamicServerFilter(); + this.afterJoinFilter = explainPlanAttributes.getAfterJoinFilter(); + this.joinScannerLimit = explainPlanAttributes.getJoinScannerLimit(); + this.sortMergeSkipMerge = explainPlanAttributes.isSortMergeSkipMerge(); this.regionLocations = explainPlanAttributes.getRegionLocations(); + this.regionLocationsTotalSize = explainPlanAttributes.getRegionLocationsTotalSize(); this.numRegionLocationLookups = explainPlanAttributes.getNumRegionLocationLookups(); } - public ExplainPlanAttributesBuilder setAbstractExplainPlan(String abstractExplainPlan) { - this.abstractExplainPlan = abstractExplainPlan; + public ExplainPlanAttributesBuilder setTenantId(String tenantId) { + this.tenantId = tenantId; return this; } - public ExplainPlanAttributesBuilder setSplitsChunk(Integer splitsChunk) { - this.splitsChunk = splitsChunk; + public ExplainPlanAttributesBuilder setViewName(String viewName) { + this.viewName = viewName; + return this; + } + + public ExplainPlanAttributesBuilder setViewBaseName(String viewBaseName) { + this.viewBaseName = viewBaseName; + return this; + } + + public ExplainPlanAttributesBuilder setCdcScopes(String cdcScopes) { + this.cdcScopes = cdcScopes; + return this; + } + + public ExplainPlanAttributesBuilder setTxnProvider(String txnProvider) { + this.txnProvider = txnProvider; + return this; + } + + public ExplainPlanAttributesBuilder setRewrites(List rewrites) { + this.rewrites = rewrites == null ? null : new ArrayList<>(rewrites); + return this; + } + + public ExplainPlanAttributesBuilder addRewrite(String rewrite) { + if (this.rewrites == null) { + this.rewrites = new ArrayList<>(); + } + this.rewrites.add(rewrite); return this; } @@ -416,28 +803,29 @@ public ExplainPlanAttributesBuilder setEstimatedSizeInBytes(Long estimatedSizeIn return this; } - public ExplainPlanAttributesBuilder setIteratorTypeAndScanSize(String iteratorTypeAndScanSize) { - this.iteratorTypeAndScanSize = iteratorTypeAndScanSize; + public ExplainPlanAttributesBuilder setEstimateInfoTs(Long estimateInfoTs) { + this.estimateInfoTs = estimateInfoTs; return this; } - public ExplainPlanAttributesBuilder setSamplingRate(Double samplingRate) { - this.samplingRate = samplingRate; + public ExplainPlanAttributesBuilder setAbstractExplainPlan(String abstractExplainPlan) { + this.abstractExplainPlan = abstractExplainPlan; return this; } - public ExplainPlanAttributesBuilder setUseRoundRobinIterator(boolean useRoundRobinIterator) { - this.useRoundRobinIterator = useRoundRobinIterator; + public ExplainPlanAttributesBuilder + setOnDuplicateKeyAction(OnDuplicateKeyType onDuplicateKeyAction) { + this.onDuplicateKeyAction = onDuplicateKeyAction; return this; } - public ExplainPlanAttributesBuilder setHexStringRVCOffset(String hexStringRVCOffset) { - this.hexStringRVCOffset = hexStringRVCOffset; + public ExplainPlanAttributesBuilder setServerUpdateSet(List serverUpdateSet) { + this.serverUpdateSet = serverUpdateSet == null ? null : new ArrayList<>(serverUpdateSet); return this; } - public ExplainPlanAttributesBuilder setConsistency(Consistency consistency) { - this.consistency = consistency; + public ExplainPlanAttributesBuilder setReturningRow(boolean returningRow) { + this.returningRow = returningRow; return this; } @@ -446,13 +834,13 @@ public ExplainPlanAttributesBuilder setHint(HintNode.Hint hint) { return this; } - public ExplainPlanAttributesBuilder setServerSortedBy(String serverSortedBy) { - this.serverSortedBy = serverSortedBy; + public ExplainPlanAttributesBuilder setExplainScanType(String explainScanType) { + this.explainScanType = explainScanType; return this; } - public ExplainPlanAttributesBuilder setExplainScanType(String explainScanType) { - this.explainScanType = explainScanType; + public ExplainPlanAttributesBuilder setConsistency(Consistency consistency) { + this.consistency = consistency; return this; } @@ -466,6 +854,41 @@ public ExplainPlanAttributesBuilder setKeyRanges(String keyRanges) { return this; } + public ExplainPlanAttributesBuilder setIndexName(String indexName) { + this.indexName = indexName; + return this; + } + + public ExplainPlanAttributesBuilder setIndexKind(String indexKind) { + this.indexKind = indexKind; + return this; + } + + public ExplainPlanAttributesBuilder setFunctionalMatch(String functionalMatch) { + this.functionalMatch = functionalMatch; + return this; + } + + public ExplainPlanAttributesBuilder setIndexRule(String indexRule) { + this.indexRule = indexRule; + return this; + } + + public ExplainPlanAttributesBuilder setIndexRejected(List indexRejected) { + this.indexRejected = indexRejected == null ? null : new ArrayList<>(indexRejected); + return this; + } + + public ExplainPlanAttributesBuilder setSaltBuckets(Integer saltBuckets) { + this.saltBuckets = saltBuckets; + return this; + } + + public ExplainPlanAttributesBuilder setRegionsPlanned(Integer regionsPlanned) { + this.regionsPlanned = regionsPlanned; + return this; + } + public ExplainPlanAttributesBuilder setScanTimeRangeMin(Long scanTimeRangeMin) { this.scanTimeRangeMin = scanTimeRangeMin; return this; @@ -476,6 +899,41 @@ public ExplainPlanAttributesBuilder setScanTimeRangeMax(Long scanTimeRangeMax) { return this; } + public ExplainPlanAttributesBuilder setSplitsChunk(Integer splitsChunk) { + this.splitsChunk = splitsChunk; + return this; + } + + public ExplainPlanAttributesBuilder setUseRoundRobinIterator(boolean useRoundRobinIterator) { + this.useRoundRobinIterator = useRoundRobinIterator; + return this; + } + + public ExplainPlanAttributesBuilder setSamplingRate(Double samplingRate) { + this.samplingRate = samplingRate; + return this; + } + + public ExplainPlanAttributesBuilder setHexStringRVCOffset(String hexStringRVCOffset) { + this.hexStringRVCOffset = hexStringRVCOffset; + return this; + } + + public ExplainPlanAttributesBuilder setIteratorTypeAndScanSize(String iteratorTypeAndScanSize) { + this.iteratorTypeAndScanSize = iteratorTypeAndScanSize; + return this; + } + + public ExplainPlanAttributesBuilder setScanEstimatedRows(Long scanEstimatedRows) { + this.scanEstimatedRows = scanEstimatedRows; + return this; + } + + public ExplainPlanAttributesBuilder setScanEstimatedSizeInBytes(Long scanEstimatedSizeInBytes) { + this.scanEstimatedSizeInBytes = scanEstimatedSizeInBytes; + return this; + } + public ExplainPlanAttributesBuilder setServerWhereFilter(String serverWhereFilter) { this.serverWhereFilter = serverWhereFilter; return this; @@ -486,19 +944,68 @@ public ExplainPlanAttributesBuilder setServerDistinctFilter(String serverDistinc return this; } - public ExplainPlanAttributesBuilder setServerOffset(Integer serverOffset) { - this.serverOffset = serverOffset; + public ExplainPlanAttributesBuilder setServerMergeColumns(Set columns) { + this.serverMergeColumns = columns == null ? null : new LinkedHashSet<>(columns); return this; } - public ExplainPlanAttributesBuilder setServerRowLimit(Long serverRowLimit) { - this.serverRowLimit = serverRowLimit; + public ExplainPlanAttributesBuilder + setServerParsedProjections(Map> serverParsedProjections) { + this.serverParsedProjections = + serverParsedProjections == null ? null : new LinkedHashMap<>(serverParsedProjections); + return this; + } + + public ExplainPlanAttributesBuilder addServerParsedProjection(String label, + List details) { + if (this.serverParsedProjections == null) { + this.serverParsedProjections = new LinkedHashMap<>(); + } + this.serverParsedProjections.put(label, + Collections.unmodifiableList(new ArrayList<>(details))); + return this; + } + + public ExplainPlanAttributesBuilder setServerProject(List serverProject) { + this.serverProject = serverProject == null ? null : new ArrayList<>(serverProject); + return this; + } + + public ExplainPlanAttributesBuilder setServerFilters(List serverFilters) { + this.serverFilters = serverFilters == null ? null : new ArrayList<>(serverFilters); + return this; + } + + public ExplainPlanAttributesBuilder addServerFilter(ExplainFilter serverFilter) { + if (this.serverFilters == null) { + this.serverFilters = new ArrayList<>(); + } + this.serverFilters.add(serverFilter); + return this; + } + + public ExplainPlanAttributesBuilder setIgnoredHints(Map ignoredHints) { + this.ignoredHints = ignoredHints == null ? null : new LinkedHashMap<>(ignoredHints); + return this; + } + + public ExplainPlanAttributesBuilder addIgnoredHint(String hint, String reason) { + if (this.ignoredHints == null) { + this.ignoredHints = new LinkedHashMap<>(); + } + this.ignoredHints.put(hint, reason); + return this; + } + + public ExplainPlanAttributesBuilder + setServerFirstKeyOnlyProjection(boolean serverFirstKeyOnlyProjection) { + this.serverFirstKeyOnlyProjection = serverFirstKeyOnlyProjection; return this; } public ExplainPlanAttributesBuilder - setServerArrayElementProjection(boolean serverArrayElementProjection) { - this.serverArrayElementProjection = serverArrayElementProjection; + setServerEmptyColumnOnlyProjection(boolean serverEmptyColumnOnlyProjection) { + this.serverEmptyColumnOnlyProjection = serverEmptyColumnOnlyProjection; return this; } @@ -507,18 +1014,51 @@ public ExplainPlanAttributesBuilder setServerAggregate(String serverAggregate) { return this; } + public ExplainPlanAttributesBuilder setServerGroupByLimit(Integer serverGroupByLimit) { + this.serverGroupByLimit = serverGroupByLimit; + return this; + } + + public ExplainPlanAttributesBuilder setServerSortedBy(String serverSortedBy) { + this.serverSortedBy = serverSortedBy; + return this; + } + + public ExplainPlanAttributesBuilder setServerOffset(Integer serverOffset) { + this.serverOffset = serverOffset; + return this; + } + + public ExplainPlanAttributesBuilder setServerRowLimit(Long serverRowLimit) { + this.serverRowLimit = serverRowLimit; + return this; + } + public ExplainPlanAttributesBuilder setClientFilterBy(String clientFilterBy) { this.clientFilterBy = clientFilterBy; return this; } + public ExplainPlanAttributesBuilder setClientFilters(List clientFilters) { + this.clientFilters = clientFilters == null ? null : new ArrayList<>(clientFilters); + return this; + } + + public ExplainPlanAttributesBuilder addClientFilter(ExplainFilter clientFilter) { + if (this.clientFilters == null) { + this.clientFilters = new ArrayList<>(); + } + this.clientFilters.add(clientFilter); + return this; + } + public ExplainPlanAttributesBuilder setClientAggregate(String clientAggregate) { this.clientAggregate = clientAggregate; return this; } - public ExplainPlanAttributesBuilder setClientSortedBy(String clientSortedBy) { - this.clientSortedBy = clientSortedBy; + public ExplainPlanAttributesBuilder setClientDistinctFilter(String clientDistinctFilter) { + this.clientDistinctFilter = clientDistinctFilter; return this; } @@ -527,8 +1067,13 @@ public ExplainPlanAttributesBuilder setClientAfterAggregate(String clientAfterAg return this; } - public ExplainPlanAttributesBuilder setClientDistinctFilter(String clientDistinctFilter) { - this.clientDistinctFilter = clientDistinctFilter; + public ExplainPlanAttributesBuilder setClientSortAlgo(String clientSortAlgo) { + this.clientSortAlgo = clientSortAlgo; + return this; + } + + public ExplainPlanAttributesBuilder setClientSortedBy(String clientSortedBy) { + this.clientSortedBy = clientSortedBy; return this; } @@ -552,8 +1097,22 @@ public ExplainPlanAttributesBuilder setClientCursorName(String clientCursorName) return this; } - public ExplainPlanAttributesBuilder setClientSortAlgo(String clientSortAlgo) { - this.clientSortAlgo = clientSortAlgo; + public ExplainPlanAttributesBuilder setClientSteps(List clientSteps) { + this.clientSteps = clientSteps == null ? null : new ArrayList<>(clientSteps); + return this; + } + + public ExplainPlanAttributesBuilder addClientStep(String step) { + if (this.clientSteps == null) { + this.clientSteps = new ArrayList<>(); + } + this.clientSteps.add(step); + return this; + } + + public ExplainPlanAttributesBuilder + setLhsJoinQueryExplainPlan(ExplainPlanAttributes lhsJoinQueryExplainPlan) { + this.lhsJoinQueryExplainPlan = lhsJoinQueryExplainPlan; return this; } @@ -563,8 +1122,28 @@ public ExplainPlanAttributesBuilder setClientSortAlgo(String clientSortAlgo) { return this; } - public ExplainPlanAttributesBuilder setServerMergeColumns(Set columns) { - this.serverMergeColumns = columns; + public ExplainPlanAttributesBuilder setSubPlans(List subPlans) { + this.subPlans = subPlans; + return this; + } + + public ExplainPlanAttributesBuilder setDynamicServerFilter(String dynamicServerFilter) { + this.dynamicServerFilter = dynamicServerFilter; + return this; + } + + public ExplainPlanAttributesBuilder setAfterJoinFilter(String afterJoinFilter) { + this.afterJoinFilter = afterJoinFilter; + return this; + } + + public ExplainPlanAttributesBuilder setJoinScannerLimit(Long joinScannerLimit) { + this.joinScannerLimit = joinScannerLimit; + return this; + } + + public ExplainPlanAttributesBuilder setSortMergeSkipMerge(boolean sortMergeSkipMerge) { + this.sortMergeSkipMerge = sortMergeSkipMerge; return this; } @@ -573,20 +1152,19 @@ public ExplainPlanAttributesBuilder setRegionLocations(List reg return this; } + public ExplainPlanAttributesBuilder + setRegionLocationsTotalSize(Integer regionLocationsTotalSize) { + this.regionLocationsTotalSize = regionLocationsTotalSize; + return this; + } + public ExplainPlanAttributesBuilder setNumRegionLocationLookups(int numRegionLocationLookups) { this.numRegionLocationLookups = numRegionLocationLookups; return this; } public ExplainPlanAttributes build() { - return new ExplainPlanAttributes(abstractExplainPlan, splitsChunk, estimatedRows, - estimatedSizeInBytes, iteratorTypeAndScanSize, samplingRate, useRoundRobinIterator, - hexStringRVCOffset, consistency, hint, serverSortedBy, explainScanType, tableName, - keyRanges, scanTimeRangeMin, scanTimeRangeMax, serverWhereFilter, serverDistinctFilter, - serverOffset, serverRowLimit, serverArrayElementProjection, serverAggregate, clientFilterBy, - clientAggregate, clientSortedBy, clientAfterAggregate, clientDistinctFilter, clientOffset, - clientRowLimit, clientSequenceCount, clientCursorName, clientSortAlgo, - rhsJoinQueryExplainPlan, serverMergeColumns, regionLocations, numRegionLocationLookups); + return new ExplainPlanAttributes(this); } } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java index 7b090467b3a..06db0584182 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/GroupByCompiler.java @@ -164,6 +164,7 @@ public List getOrderPreservingTrackInfos() { return orderPreservingTrackInfos; } + @SuppressWarnings("rawtypes") public GroupBy compile(StatementContext context, QueryPlan innerQueryPlan, Expression whereExpression) throws SQLException { boolean isOrderPreserving = this.isOrderPreserving; @@ -365,6 +366,9 @@ private void explainUtil(List planSteps, Integer limit, planSteps.add(" " + serverAggregate); if (explainPlanAttributesBuilder != null) { explainPlanAttributesBuilder.setServerAggregate(serverAggregate); + if (!isUngroupedAggregate && limit != null) { + explainPlanAttributesBuilder.setServerGroupByLimit(limit); + } } } @@ -448,6 +452,7 @@ public static GroupBy compile(StatementContext context, SelectStatement statemen return groupBy; } + @SuppressWarnings("rawtypes") private static boolean onlyAtEndType(Expression expression) { // Due to the encoding schema of these types, they may only be // used once in a group by and are located at the end of the @@ -456,6 +461,7 @@ private static boolean onlyAtEndType(Expression expression) { return type.isArrayType() || type == PVarbinary.INSTANCE; } + @SuppressWarnings("rawtypes") private static PDataType getGroupByDataType(Expression expression) { return IndexUtil.getIndexColumnDataType(expression.isNullable(), expression.getDataType()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/HavingCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/HavingCompiler.java index f57a126f741..01b39058204 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/HavingCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/HavingCompiler.java @@ -64,6 +64,14 @@ public static Expression compile(StatementContext context, SelectStatement state throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_AGGREGATE_IN_HAVING_CLAUSE).build() .buildException(); } + // Tag the residual HAVING predicate(s) with their origin for VERBOSE attribution. + if (expression instanceof org.apache.phoenix.expression.AndExpression) { + for (Expression child : expression.getChildren()) { + context.tagPredicate(child, "HAVING"); + } + } else { + context.tagPredicate(expression, "HAVING"); + } return expression; } @@ -75,6 +83,9 @@ public static SelectStatement rewrite(StatementContext context, SelectStatement } HavingClauseVisitor visitor = new HavingClauseVisitor(context, groupBy); having.accept(visitor); + if (!visitor.getMoveToWhereClauseExpressions().isEmpty()) { + context.addAppliedRewrite("HAVING PREDICATE AS WHERE"); + } statement = SelectStatementRewriter.moveFromHavingToWhereClause(statement, visitor.getMoveToWhereClauseExpressions()); return statement; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index 8a091fbdc06..5beab671c85 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -128,9 +128,10 @@ public enum ColumnRefType { private final Map columnRefs; private final Map columnNodes; private final boolean useSortMergeJoin; + private final StatementContext context; - private JoinCompiler(PhoenixStatement statement, SelectStatement select, - ColumnResolver resolver) { + private JoinCompiler(PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, + StatementContext context) { this.phoenixStatement = statement; this.originalJoinSelectStatement = select; this.origResolver = resolver; @@ -138,16 +139,25 @@ private JoinCompiler(PhoenixStatement statement, SelectStatement select, this.columnRefs = new HashMap(); this.columnNodes = new HashMap(); this.useSortMergeJoin = select.getHint().hasHint(Hint.USE_SORT_MERGE_JOIN); + this.context = context; + } + + public static JoinTable compile(PhoenixStatement statement, SelectStatement select, + ColumnResolver resolver) throws SQLException { + return compile(statement, select, resolver, null); } /** * After this method is called, the inner state of the parameter resolver may be changed by * {@link FromCompiler#refreshDerivedTableNode} because of some sql optimization, see also * {@link Table#pruneSubselectAliasedNodes()}. + *

+ * The supplied {@link StatementContext} receives top of plan rewrite breadcrumbs recorded during + * join compilation. May be null. */ public static JoinTable compile(PhoenixStatement statement, SelectStatement select, - ColumnResolver resolver) throws SQLException { - JoinCompiler compiler = new JoinCompiler(statement, select, resolver); + ColumnResolver resolver, StatementContext context) throws SQLException { + JoinCompiler compiler = new JoinCompiler(statement, select, resolver, context); JoinTableConstructor constructor = compiler.new JoinTableConstructor(); Pair> res = select.getFrom().accept(constructor); JoinTable joinTable = res.getSecond() == null @@ -253,7 +263,7 @@ public class JoinTable { private JoinTable(Table table) { this.leftTable = table; this.joinSpecs = Collections. emptyList(); - this.postFilters = Collections.EMPTY_LIST; + this.postFilters = Collections.emptyList(); this.allTables = Collections. singletonList(table); this.allTableRefs = Collections. singletonList(table.getTableRef()); this.allLeftJoin = false; @@ -479,6 +489,10 @@ public boolean[] getStarJoinVector() throws SQLException { } } + if (context != null && vector.length >= 2) { + context.addAppliedRewrite("STAR JOIN ON " + vector.length + " RIGHT LEGS"); + } + return vector; } @@ -633,6 +647,7 @@ public Set getDependentTableRefs() { return dependentTableRefs; } + @SuppressWarnings("rawtypes") public Pair, List> compileJoinConditions(StatementContext lhsCtx, StatementContext rhsCtx, Strategy strategy) throws SQLException { if (onConditions.isEmpty()) { @@ -660,6 +675,12 @@ Collections. singletonList(LiteralExpression.newConstant(1)), if (right.getDataType() != toType || right.getSortOrder() != toSortOrder) { right = CoerceExpression.create(right, toType, toSortOrder, right.getMaxLength()); } + // Tag the compiled ON predicates with their origin for VERBOSE attribution. + String decorrelatedAlias = lhsCtx.getDecorrelatedSubqueryAlias(condition); + String onOrigin = + decorrelatedAlias == null ? "JOIN ON" : "decorrelated from " + decorrelatedAlias; + lhsCtx.tagPredicate(left, onOrigin); + rhsCtx.tagPredicate(right, onOrigin); compiled.add(new Pair(left, right)); } // TODO PHOENIX-4618: @@ -704,6 +725,7 @@ public int compare(Pair o1, Pair return new Pair, List>(lConditions, rConditions); } + @SuppressWarnings("rawtypes") private PDataType getCommonType(PDataType lType, PDataType rType) throws SQLException { if (lType == rType) return lType; @@ -823,7 +845,7 @@ private Table(DerivedTableNode tableNode, boolean isWildcard, TableRef tableRef) this.dynamicColumns = Collections. emptyList(); this.tableSamplingRate = ConcreteTableNode.DEFAULT_TABLE_SAMPLING_RATE; this.subselectStatement = - SubselectRewriter.flatten(tableNode.getSelect(), phoenixStatement.getConnection()); + SubselectRewriter.flatten(tableNode.getSelect(), phoenixStatement.getConnection(), context); this.tableRef = tableRef; this.preFilterParseNodes = new ArrayList(); this.postFilterParseNodes = new ArrayList(); @@ -1056,8 +1078,10 @@ public boolean isCouldPushToServerAsHashJoinProbeSide() throws SQLException { } SelectStatement selectStatementToUse = this.getAsSubquery(null); + // The rewrite result is only used to test a predicate, so any rewrite breadcrumbs are + // recorded on a throwaway context and discarded. RewriteResult rewriteResult = - ParseNodeUtil.rewrite(selectStatementToUse, phoenixStatement.getConnection()); + ParseNodeUtil.rewrite(selectStatementToUse, new StatementContext(phoenixStatement)); return JoinCompiler .isCouldPushToServerAsHashJoinProbeSide(rewriteResult.getRewrittenSelectStatement()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java index 5630d22ec69..101eb035109 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/OrderByCompiler.java @@ -224,6 +224,7 @@ public static OrderBy compile(StatementContext context, SelectStatement statemen if (offset.getByteOffset().isPresent()) { throw new SQLException("Do not allow non-pk ORDER BY with RVC OFFSET"); } + context.addAppliedRewrite("REVERSE SCAN SUBSTITUTION"); return OrderBy.REV_ROW_KEY_ORDER_BY; } } else { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java index 4ee8c1f4692..138f000e508 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java @@ -600,6 +600,9 @@ public static RowProjector compile(StatementContext context, SelectStatement sta serverAttributeToKVExpressionMap.get(entry.getKey())); } } + // Stash the per-type expression buckets on the context so EXPLAIN can render the per-type + // SERVER ARRAY|JSON|BSON PROJECTION clauses (and their per-expression detail lines). + context.setServerParsedProjections(serverAttributeToFuncExpressionMap); KeyValueSchemaBuilder builder = new KeyValueSchemaBuilder(0); for (Expression expression : serverParsedKVRefs) { builder.addField(expression); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java index f4d78ce0e69..8b187042aa0 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java @@ -121,6 +121,7 @@ public class QueryCompiler { private final Map dataPlans; private final boolean costBased; private final StatementContext parentContext; + private StatementContext prebuiltContext; public QueryCompiler(PhoenixStatement statement, SelectStatement select, ColumnResolver resolver, boolean projectTuples, boolean optimizeSubquery, Map dataPlans) @@ -268,6 +269,9 @@ public QueryPlan compileUnionAll(SelectStatement select) throws SQLException { ColumnResolver resolver = FromCompiler.getResolver(tableRef); StatementContext context = new StatementContext(statement, resolver, bindManager, scan, sequenceManager); + if (prebuiltContext != null) { + context.adoptRewriteState(prebuiltContext); + } plans = UnionCompiler.convertToTupleProjectionPlan(plans, tableRef, context); QueryPlan plan = compileSingleFlatQuery(context, select, false, false, null, false, true); plan = new UnionPlan(context, select, tableRef, plan.getProjector(), plan.getLimit(), @@ -290,7 +294,7 @@ private QueryPlan getExistingDataPlanForCDC() { public QueryPlan compileSelect(SelectStatement select) throws SQLException { StatementContext context = createStatementContext(); if (parentContext != null) { - parentContext.addSubStatementContext(context); + parentContext.addSubStatementContext(context, false); } QueryPlan dataPlanForCDC = getExistingDataPlanForCDC(); if (dataPlanForCDC != null) { @@ -313,15 +317,34 @@ public QueryPlan compileSelect(SelectStatement select) throws SQLException { context.setCDCIncludeScopes(cdcIncludeScopes); } if (select.isJoin()) { - JoinTable joinTable = JoinCompiler.compile(statement, select, context.getResolver()); + JoinTable joinTable = JoinCompiler.compile(statement, select, context.getResolver(), context); return compileJoinQuery(context, joinTable, false, false, null); } else { + // A USE_SORT_MERGE_JOIN hint on a query without any join is ignored. + if (select.getHint().hasHint(Hint.USE_SORT_MERGE_JOIN)) { + context.recordIgnoredHint(Hint.USE_SORT_MERGE_JOIN, "no join in query"); + } return compileSingleQuery(context, select, false, true); } } + /** + * Supplies the pre-built top level {@link StatementContext} used by + * {@link org.apache.phoenix.util.ParseNodeUtil#rewrite} so the breadcrumbs recorded by the early + * rewrite pass are carried onto the compilation context. + */ + public QueryCompiler withRewriteContext(StatementContext prebuiltContext) { + this.prebuiltContext = prebuiltContext; + return this; + } + private StatementContext createStatementContext() { - return new StatementContext(statement, resolver, bindManager, scan, sequenceManager); + StatementContext context = + new StatementContext(statement, resolver, bindManager, scan, sequenceManager); + if (prebuiltContext != null) { + context.adoptRewriteState(prebuiltContext); + } + return context; } /** @@ -384,6 +407,7 @@ protected QueryPlan compileJoinQuery(StatementContext context, JoinTable joinTab return bestPlan; } + @SuppressWarnings("unchecked") protected QueryPlan compileJoinQuery(JoinCompiler.Strategy strategy, StatementContext context, JoinTable joinTable, boolean asSubquery, boolean projectPKColumns, List orderBy) throws SQLException { @@ -486,11 +510,14 @@ protected QueryPlan compileJoinQuery(JoinCompiler.Strategy strategy, StatementCo joinTypes, starJoinVector, tables, fieldPositions, postJoinFilterExpression, QueryUtil.getOffsetLimit(limit, offset)); return HashJoinPlan.create(joinTable.getOriginalJoinSelectStatement(), plan, joinInfo, - hashPlans); + hashPlans, JoinCompiler.Strategy.HASH_BUILD_RIGHT); } case HASH_BUILD_LEFT: { JoinSpec lastJoinSpec = joinSpecs.get(joinSpecs.size() - 1); JoinType type = lastJoinSpec.getType(); + if (type == JoinType.Right) { + context.addAppliedRewrite("RIGHT JOIN AS LEFT JOIN"); + } JoinTable rhsJoinTable = lastJoinSpec.getRhsJoinTable(); Table rhsTable = rhsJoinTable.getLeftTable(); JoinTable lhsJoin = joinTable.createSubJoinTable(statement.getConnection()); @@ -562,7 +589,8 @@ protected QueryPlan compileJoinQuery(JoinCompiler.Strategy strategy, StatementCo hashExpressions); return HashJoinPlan.create(joinTable.getOriginalJoinSelectStatement(), rhsPlan, joinInfo, new HashSubPlan[] { new HashSubPlan(0, lhsPlan, hashExpressions, false, - usePersistentCache, keyRangeExpressions.getFirst(), keyRangeExpressions.getSecond()) }); + usePersistentCache, keyRangeExpressions.getFirst(), keyRangeExpressions.getSecond()) }, + JoinCompiler.Strategy.HASH_BUILD_LEFT); } case SORT_MERGE: { JoinTable lhsJoin = joinTable.createSubJoinTable(statement.getConnection()); @@ -570,6 +598,7 @@ protected QueryPlan compileJoinQuery(JoinCompiler.Strategy strategy, StatementCo JoinType type = lastJoinSpec.getType(); JoinTable rhsJoin = lastJoinSpec.getRhsJoinTable(); if (type == JoinType.Right) { + context.addAppliedRewrite("RIGHT JOIN AS LEFT JOIN"); JoinTable temp = lhsJoin; lhsJoin = rhsJoin; rhsJoin = temp; @@ -695,14 +724,15 @@ protected QueryPlan compileSubquery(SelectStatement subquery, boolean pushDownMa protected QueryPlan compileSubquery(SelectStatement subquerySelectStatement, boolean pushDownMaxRows, StatementContext parentContext) throws SQLException { - PhoenixConnection phoenixConnection = this.statement.getConnection(); - RewriteResult rewriteResult = ParseNodeUtil.rewrite(subquerySelectStatement, phoenixConnection); + StatementContext rewriteContext = StatementContext.forRewrite(this.statement, bindManager); + RewriteResult rewriteResult = ParseNodeUtil.rewrite(subquerySelectStatement, rewriteContext); int maxRows = this.statement.getMaxRows(); - this.statement.setMaxRows(pushDownMaxRows ? maxRows : 0); // overwrite maxRows to avoid its - // impact on inner queries. + // overwrite maxRows to avoid its impact on inner queries. + this.statement.setMaxRows(pushDownMaxRows ? maxRows : 0); QueryPlan queryPlan = new QueryCompiler(this.statement, rewriteResult.getRewrittenSelectStatement(), - rewriteResult.getColumnResolver(), bindManager, false, optimizeSubquery, null).compile(); + rewriteResult.getColumnResolver(), bindManager, false, optimizeSubquery, null) + .withRewriteContext(rewriteContext).compile(); if (optimizeSubquery) { queryPlan = statement.getConnection().getQueryServices().getOptimizer().optimize(statement, queryPlan); @@ -732,7 +762,7 @@ protected QueryPlan compileSingleQuery(StatementContext context, SelectStatement QueryPlan innerPlan = compileSubquery(innerSelect, false, context); if (innerPlan instanceof UnionPlan) { UnionCompiler.optimizeUnionOrderByIfPossible((UnionPlan) innerPlan, select, - this::createStatementContext); + this::createStatementContext, context); } RowProjector innerQueryPlanRowProjector = innerPlan.getProjector(); TupleProjector tupleProjector = new TupleProjector(innerQueryPlanRowProjector); @@ -895,7 +925,7 @@ protected QueryPlan compileSingleFlatQuery(StatementContext context, SelectState subPlans[i++] = new WhereClauseSubPlan(compileSubquery(stmt, false), stmt, subqueryNode.expectSingleRow()); } - plan = HashJoinPlan.create(planSelect, plan, null, subPlans); + plan = HashJoinPlan.create(planSelect, plan, null, subPlans, null); } if (innerPlan != null) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java index a56a262b808..1b2d315c9b8 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java @@ -28,6 +28,7 @@ import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.optimize.Cost; +import org.apache.phoenix.optimize.OptimizerDecision; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.parse.SelectStatement; import org.apache.phoenix.query.KeyRange; @@ -103,4 +104,17 @@ public interface QueryPlan extends StatementPlan { * */ public List getOutputOrderBys(); + + /** + * The optimizer's index selection rationale for this plan, or {@code null} if this plan did not + * participate in optimizer index selection. + */ + default OptimizerDecision getOptimizerDecision() { + return null; + } + + /** Records the optimizer's index selection rationale on this plan. */ + default void setOptimizerDecision(OptimizerDecision decision) { + // no-op + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RVCOffsetCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RVCOffsetCompiler.java index 739f7f019a5..abb3270017f 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RVCOffsetCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RVCOffsetCompiler.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Set; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.expression.AndExpression; import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; @@ -179,6 +180,9 @@ public CompiledOffset getRVCOffset(StatementContext context, FilterableStatement throw new RowValueConstructorOffsetInternalErrorException("RVC Offset unexpected failure."); } + // Tag the RVC offset predicate with its origin for VERBOSE attribution. + context.tagPredicate(whereExpression, "RVC OFFSET"); + Expression expression; try { expression = WhereOptimizer.pushKeyExpressionsToScan(context, originalHints, whereExpression, @@ -289,6 +293,8 @@ public CompiledOffset getRVCOffset(StatementContext context, FilterableStatement CompiledOffset compiledOffset = new CompiledOffset(Optional. absent(), Optional.of(key)); + context.addAppliedRewrite("RVC OFFSET 0x" + Bytes.toHex(key)); + return compiledOffset; } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RegionLocationsListSerializer.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RegionLocationsListSerializer.java new file mode 100644 index 00000000000..1ae1b77a534 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RegionLocationsListSerializer.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compile; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Jackson serializer for {@code List} as it appears on + * {@link ExplainPlanAttributes#getRegionLocations()}. The HBase {@code HRegionLocation} bean is not + * cleanly serializable by Jackson's default introspection. + */ +public class RegionLocationsListSerializer extends StdSerializer> { + + private static final long serialVersionUID = 1L; + + @SuppressWarnings("unchecked") + public RegionLocationsListSerializer() { + super((Class>) (Class) List.class); + } + + @Override + public void serialize(List value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + gen.writeStartArray(); + for (HRegionLocation loc : value) { + gen.writeStartObject(); + RegionInfo region = loc == null ? null : loc.getRegion(); + gen.writeStringField("startKey", + region == null ? null : Bytes.toStringBinary(region.getStartKey())); + gen.writeStringField("endKey", + region == null ? null : Bytes.toStringBinary(region.getEndKey())); + ServerName sn = loc == null ? null : loc.getServerName(); + gen.writeStringField("server", sn == null ? null : sn.toString()); + gen.writeEndObject(); + } + gen.writeEndArray(); + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ServerMergeColumnsSerializer.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ServerMergeColumnsSerializer.java new file mode 100644 index 00000000000..988c319bb85 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ServerMergeColumnsSerializer.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compile; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import org.apache.phoenix.schema.PColumn; + +/** + * Jackson serializer for {@code Set} as it appears on + * {@link ExplainPlanAttributes#getServerMergeColumns()}. {@code PColumn} is an interface backed by + * implementations that are not Jackson-friendly. + */ +public class ServerMergeColumnsSerializer extends StdSerializer> { + + private static final long serialVersionUID = 1L; + + @SuppressWarnings("unchecked") + public ServerMergeColumnsSerializer() { + super((Class>) (Class) Set.class); + } + + @Override + public void serialize(Set value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + List names = new ArrayList<>(value.size()); + for (PColumn column : value) { + names.add(column == null ? null : column.toString()); + } + Collections.sort(names, Comparator.nullsFirst(Comparator.naturalOrder())); + gen.writeStartArray(); + for (String name : names) { + gen.writeString(name); + } + gen.writeEndArray(); + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java index 1053dac1ea9..6767035b3c3 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java @@ -21,9 +21,14 @@ import java.text.Format; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Deque; +import java.util.EnumMap; +import java.util.IdentityHashMap; import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,6 +36,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.expression.Expression; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.log.QueryLogger; @@ -39,6 +45,9 @@ import org.apache.phoenix.monitoring.ScanMetricsHolder; import org.apache.phoenix.monitoring.SlowestScanMetricsQueue; import org.apache.phoenix.monitoring.TopNTreeMultiMap; +import org.apache.phoenix.parse.ExplainOptions; +import org.apache.phoenix.parse.HintNode.Hint; +import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.SelectStatement; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; @@ -101,11 +110,40 @@ public class StatementContext { private boolean hasRawRowSizeFunction = false; private final SlowestScanMetricsQueue slowestScanMetricsQueue; private final int slowestScanMetricsCount; + private List appliedRewrites; + private int derivedTableFlattenCount; + private Map> appliedIndexExpressionMatches; + private Map> appliedIndexExpressionPairs; + private Set functionalIndexNames; + private Set> partialIndexCheckedSet; + private Map> serverParsedProjections; + private StatementContext parentContext; + private ExplainOptions explainOptions = ExplainOptions.DEFAULT; + private Map> predicateOrigins; + private Map decorrelatedSubqueryAlias; + private Map ignoredHints; public StatementContext(PhoenixStatement statement) { this(statement, new Scan()); } + /** + * Build a top-level {@link StatementContext} suitable for the early + * {@link SubselectRewriter}/{@link SubqueryRewriter} rewrite pass, accumulating rewrite + * breadcrumbs. + */ + public static StatementContext forRewrite(PhoenixStatement statement) { + return forRewrite(statement, new BindManager(statement.getParameters())); + } + + /** + * Variant of {@link #forRewrite(PhoenixStatement)} that reuses an existing {@link BindManager}. + */ + public static StatementContext forRewrite(PhoenixStatement statement, BindManager bindManager) { + return new StatementContext(statement, FromCompiler.EMPTY_TABLE_RESOLVER, bindManager, + new Scan(), new SequenceManager(statement)); + } + public StatementContext(StatementContext context) { this.resolver = context.resolver; this.connection = context.connection; @@ -132,11 +170,14 @@ public StatementContext(StatementContext context) { this.isClientSideUpsertSelect = context.isClientSideUpsertSelect; this.isUncoveredIndex = context.isUncoveredIndex; this.hasFirstValidResult = new AtomicBoolean(context.getHasFirstValidResult()); - this.subStatementContexts = Sets.newHashSet(); + this.subStatementContexts = Sets.newLinkedHashSet(); this.totalSegmentsFunction = context.totalSegmentsFunction; this.totalSegmentsValue = context.totalSegmentsValue; this.hasRowSizeFunction = context.hasRowSizeFunction; this.hasRawRowSizeFunction = context.hasRawRowSizeFunction; + this.parentContext = context.parentContext; + this.explainOptions = context.explainOptions; + copyRewriteStateFrom(context); } /** @@ -198,7 +239,18 @@ public StatementContext(PhoenixStatement statement, ColumnResolver resolver, Bin : SlowestScanMetricsQueue.NOOP_SLOWEST_SCAN_METRICS_QUEUE; this.retryingPersistentCache = Maps. newHashMap(); this.hasFirstValidResult = new AtomicBoolean(false); - this.subStatementContexts = Sets.newHashSet(); + this.subStatementContexts = Sets.newLinkedHashSet(); + this.appliedRewrites = new ArrayList<>(); + this.derivedTableFlattenCount = 0; + this.appliedIndexExpressionMatches = Maps.newLinkedHashMap(); + this.appliedIndexExpressionPairs = Maps.newLinkedHashMap(); + this.functionalIndexNames = Sets.newLinkedHashSet(); + this.partialIndexCheckedSet = Sets.newLinkedHashSet(); + this.serverParsedProjections = null; + this.parentContext = null; + this.predicateOrigins = new IdentityHashMap<>(); + this.decorrelatedSubqueryAlias = new IdentityHashMap<>(); + this.ignoredHints = new EnumMap<>(Hint.class); } /** @@ -453,13 +505,260 @@ public void setCDCTableRef(TableRef cdcTableRef) { } public void addSubStatementContext(StatementContext sub) { + addSubStatementContext(sub, true); + } + + public void addSubStatementContext(StatementContext sub, boolean linkParentForDisclosure) { subStatementContexts.add(sub); + if (linkParentForDisclosure) { + sub.parentContext = this; + } } public Set getSubStatementContexts() { return subStatementContexts; } + /** + * Records a top-of-plan rewrite breadcrumb (e.g. "STAR JOIN ON 2 RIGHT LEGS"). Breadcrumbs are + * diagnostic only and never affect the compiled plan. + */ + public void addAppliedRewrite(String rewrite) { + appliedRewrites.add(rewrite); + } + + public List getAppliedRewrites() { + return appliedRewrites; + } + + /** + * Adopt the rewrite breadcrumb accumulator (and related diagnostic state) of another context. + * Used to carry breadcrumbs recorded by the early + * {@link SubselectRewriter}/{@link SubqueryRewriter} pass on a pre-built context across the + * rewrite boundary onto the actual compilation context. Fresh collections are allocated (see + * {@link #copyRewriteStateFrom}) so this context and {@code source} continue to mutate + * independently. + */ + public void adoptRewriteState(StatementContext source) { + copyRewriteStateFrom(source); + } + + /** + * Copy the rewrite breadcrumb accumulators and related diagnostic state from {@code source} into + * this context, allocating fresh collections so the two contexts never share mutable backing + * collections. Shared by the copy constructor and {@link #adoptRewriteState}, both of which hand + * the resulting context to a compile pass that continues to mutate this state. + */ + private void copyRewriteStateFrom(StatementContext source) { + this.appliedRewrites = + source.appliedRewrites == null ? new ArrayList<>() : new ArrayList<>(source.appliedRewrites); + this.derivedTableFlattenCount = source.derivedTableFlattenCount; + this.appliedIndexExpressionMatches = Maps.newLinkedHashMap(); + for (Map.Entry> entry : source.appliedIndexExpressionMatches.entrySet()) { + this.appliedIndexExpressionMatches.put(entry.getKey(), new ArrayList<>(entry.getValue())); + } + this.appliedIndexExpressionPairs = Maps.newLinkedHashMap(); + for (Map.Entry> entry : source.appliedIndexExpressionPairs + .entrySet()) { + this.appliedIndexExpressionPairs.put(entry.getKey(), new LinkedHashMap<>(entry.getValue())); + } + this.functionalIndexNames = Sets.newLinkedHashSet(source.functionalIndexNames); + this.partialIndexCheckedSet = Sets.newLinkedHashSet(source.partialIndexCheckedSet); + // serverParsedProjections is nullable and only ever replaced wholesale via its setter (never + // mutated in place), so the reference can be carried over directly. + this.serverParsedProjections = source.serverParsedProjections; + this.predicateOrigins = new IdentityHashMap<>(); + for (Map.Entry> entry : source.predicateOrigins.entrySet()) { + this.predicateOrigins.put(entry.getKey(), new LinkedHashSet<>(entry.getValue())); + } + this.decorrelatedSubqueryAlias = new IdentityHashMap<>(source.decorrelatedSubqueryAlias); + this.ignoredHints = new EnumMap<>(source.ignoredHints); + } + + public void incrementDerivedTableFlattenCount() { + derivedTableFlattenCount++; + } + + public int getDerivedTableFlattenCount() { + return derivedTableFlattenCount; + } + + /** + * Records the path expressions that actually substituted against this query for the given + * functional index. Used by the optimizer to label the chosen index's rule as + * {@code matches } and to distinguish functional index rejections that matched no query + * expression. Deduplicated per index, preserving first seen order. + */ + public void recordAppliedIndexExpressionMatches(String indexName, + Collection expressions) { + if (expressions == null || expressions.isEmpty()) { + return; + } + List matches = + appliedIndexExpressionMatches.computeIfAbsent(indexName, k -> new ArrayList<>()); + for (String expression : expressions) { + if (!matches.contains(expression)) { + matches.add(expression); + } + } + } + + /** + * Returns the path expressions that actually substituted against this query for the given + * functional index, or an empty list if none were recorded. + */ + public List getAppliedIndexExpressionMatches(String indexName) { + List matches = appliedIndexExpressionMatches.get(indexName); + return matches == null ? Collections.emptyList() : matches; + } + + /** + * Records the {@code } pairs for substitutions that + * actually fired against this query for the given functional index. Carries enough information + * for the optimizer to emit one {@code REWRITE INDEX EXPRESSION AS } breadcrumb per + * applied substitution after the chosen plan is selected. Deduplicated per index, preserving + * first-seen insertion order. + */ + public void recordAppliedIndexExpressionPairs(String indexName, Map pairs) { + if (pairs == null || pairs.isEmpty()) { + return; + } + Map existing = + appliedIndexExpressionPairs.computeIfAbsent(indexName, k -> new LinkedHashMap<>()); + for (Map.Entry entry : pairs.entrySet()) { + existing.putIfAbsent(entry.getKey(), entry.getValue()); + } + } + + /** + * Returns the {@code } pairs that actually + * substituted against this query for the given functional index, or an empty map if none. + */ + public Map getAppliedIndexExpressionPairs(String indexName) { + Map pairs = appliedIndexExpressionPairs.get(indexName); + return pairs == null ? Collections.emptyMap() : pairs; + } + + /** Marks the given index as a functional index. */ + public void recordFunctionalIndex(String indexName) { + functionalIndexNames.add(indexName); + } + + /** Returns whether the given index was recorded as a functional index during rewriting. */ + public boolean isFunctionalIndex(String indexName) { + return functionalIndexNames.contains(indexName); + } + + /** + * Dedup set used by the optimizer so a partial index applicability breadcrumb is recorded only + * once per (table, index) pair even when the same index is scored across multiple paths. + * @return true if the pair had not been recorded before + */ + public boolean markPartialIndexChecked(String tableName, String indexName) { + return partialIndexCheckedSet.add(new Pair<>(tableName, indexName)); + } + + /** + * Server-evaluated parsed projection expressions, keyed by the scan attribute they were + * serialized into ({@code _SpecificArrayIndex}, {@code _JsonValueFunction}, + * {@code _JsonQueryFunction}, {@code _BsonValueFunction}). Populated by + * {@link ProjectionCompiler} when at least one JSON/BSON/array path expression is pushed to the + * server, and consumed by {@code ExplainTable} to render the per-type {@code SERVER * PROJECTION} + * clauses. {@code null} when no server-side parsed projection compile occurred. + */ + public Map> getServerParsedProjections() { + return serverParsedProjections; + } + + public void setServerParsedProjections(Map> serverParsedProjections) { + this.serverParsedProjections = serverParsedProjections; + } + + public StatementContext getParentContext() { + return parentContext; + } + + /** + * The options parsed from the {@code EXPLAIN} statement's option list. Defaults to + * {@link ExplainOptions#DEFAULT}. + */ + public ExplainOptions getExplainOptions() { + return explainOptions; + } + + public void setExplainOptions(ExplainOptions explainOptions) { + this.explainOptions = explainOptions == null ? ExplainOptions.DEFAULT : explainOptions; + } + + /** Returns true if the {@code EXPLAIN} statement requested {@code VERBOSE} output. */ + public boolean isVerbose() { + return explainOptions != null && explainOptions.isVerbose(); + } + + /** + * Tag a predicate {@link Expression} with the given origin (e.g. {@code "WHERE"}, + * {@code "JOIN ON"}). Tags accumulate as a set so a predicate fused from multiple sources carries + * every contributing origin. Keyed by object identity, since Phoenix rewrites and re-instantiates + * expressions during compilation. Diagnostic only; never affects the compiled plan. + */ + public void tagPredicate(Expression expression, String origin) { + if (expression == null || origin == null) { + return; + } + predicateOrigins.computeIfAbsent(expression, k -> new LinkedHashSet<>()).add(origin); + } + + /** Returns the predicate origin tags accumulated during compilation. Identity keyed. */ + public Map> getPredicateOrigins() { + return predicateOrigins; + } + + /** Returns the origin tags recorded for {@code expression}, or an empty set if none. */ + public Set getPredicateOrigins(Expression expression) { + Set tags = predicateOrigins.get(expression); + return tags == null ? Collections.emptySet() : tags; + } + + /** + * Records that the given decorrelated join-condition parse node originated from the subquery with + * the given temp alias. + */ + public void putDecorrelatedSubqueryAlias(ParseNode joinConditionNode, String subqueryAlias) { + if (joinConditionNode != null && subqueryAlias != null) { + decorrelatedSubqueryAlias.put(joinConditionNode, subqueryAlias); + } + } + + /** + * Returns the subquery temp alias the given join-condition parse node was decorrelated from, or + * {@code null} if the node is not a decorrelated predicate. + */ + public String getDecorrelatedSubqueryAlias(ParseNode joinConditionNode) { + return decorrelatedSubqueryAlias.get(joinConditionNode); + } + + /** + * Records that the planner intentionally ignored {@code hint} for the given {@code reason}. + * Surfaced under {@code EXPLAIN (VERBOSE)} as a {@code /*- HINT(args) -- reason *}{@code /} + * comment. The first reason recorded for a hint wins. Diagnostic only. + */ + public void recordIgnoredHint(Hint hint, String reason) { + if (hint == null || reason == null) { + return; + } + ignoredHints.putIfAbsent(hint, reason); + } + + /** Returns the hints the planner intentionally ignored, mapped to the reason. */ + public Map getIgnoredHints() { + return ignoredHints; + } + + /** Returns true if this is the top-level (root) statement context, i.e. it has no parent. */ + public boolean isRoot() { + return parentContext == null; + } + public boolean hasTotalSegmentsFunction() { return totalSegmentsFunction; } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubqueryRewriter.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubqueryRewriter.java index 3f7d3d22f80..fbeb389f43e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubqueryRewriter.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubqueryRewriter.java @@ -69,15 +69,16 @@ public class SubqueryRewriter extends ParseNodeRewriter { private final ColumnResolver columnResolver; private final PhoenixConnection connection; + private final StatementContext context; private TableNode tableNode; private ParseNode topNode; public static SelectStatement transform(SelectStatement select, ColumnResolver resolver, - PhoenixConnection connection) throws SQLException { + PhoenixConnection connection, StatementContext context) throws SQLException { ParseNode where = select.getWhere(); if (where == null) return select; - SubqueryRewriter rewriter = new SubqueryRewriter(select, resolver, connection); + SubqueryRewriter rewriter = new SubqueryRewriter(select, resolver, connection, context); ParseNode normWhere = rewrite(where, rewriter); if (normWhere == where) return select; @@ -85,13 +86,42 @@ public static SelectStatement transform(SelectStatement select, ColumnResolver r } protected SubqueryRewriter(SelectStatement select, ColumnResolver resolver, - PhoenixConnection connection) { + PhoenixConnection connection, StatementContext context) { this.columnResolver = resolver; this.connection = connection; + this.context = context; this.tableNode = select.getFrom(); this.topNode = null; } + /** + * Records a subquery to join rewrite breadcrumb on the active context. The wording is derived + * from the originating subquery kind and the chosen {@link JoinType}. + */ + private void recordSubqueryRewrite(String subqueryKind, JoinType joinType) { + if (context == null) { + return; + } + final String joinName; + switch (joinType) { + case Semi: + joinName = "SEMI"; + break; + case Anti: + joinName = "ANTI"; + break; + case Inner: + joinName = "INNER"; + break; + case Left: + joinName = "LEFT"; + break; + default: + joinName = joinType.name().toUpperCase(); + } + context.addAppliedRewrite(subqueryKind + " SUBQUERY AS " + joinName + " JOIN"); + } + @Override protected void enterParseNode(ParseNode node) { if (topNode == null) { @@ -193,7 +223,7 @@ public ParseNode visitLeave(InParseNode inParseNode, List childParseN String subqueryTableTempAlias = ParseNodeFactory.createTempAlias(); JoinConditionExtractor joinConditionExtractor = new JoinConditionExtractor( - subquerySelectStatementToUse, columnResolver, connection, subqueryTableTempAlias); + subquerySelectStatementToUse, columnResolver, connection, subqueryTableTempAlias, context); List newSubquerySelectAliasedNodes = null; ParseNode extractedJoinConditionParseNode = null; @@ -263,6 +293,7 @@ public ParseNode visitLeave(InParseNode inParseNode, List childParseN newSubquerySelectAliasedNodes.get(0).getAlias(), null), !inParseNode.isNegate()); tableNode = NODE_FACTORY.join(joinType, tableNode, subqueryDerivedTableNode, joinOnConditionParseNode, false); + recordSubqueryRewrite(inParseNode.isNegate() ? "NOT IN" : "IN", joinType); return resultWhereParseNode; } @@ -339,7 +370,7 @@ public ParseNode visitLeave(ExistsParseNode existsParseNode, List chi fixSubqueryStatement(subqueryParseNode.getSelectNode()); String subqueryTableTempAlias = ParseNodeFactory.createTempAlias(); JoinConditionExtractor joinConditionExtractor = new JoinConditionExtractor( - subquerySelectStatementToUse, columnResolver, connection, subqueryTableTempAlias); + subquerySelectStatementToUse, columnResolver, connection, subqueryTableTempAlias, context); ParseNode whereParseNodeAfterExtract = subquerySelectStatementToUse.getWhere() == null ? null : subquerySelectStatementToUse.getWhere().accept(joinConditionExtractor); @@ -392,6 +423,7 @@ public ParseNode visitLeave(ExistsParseNode existsParseNode, List chi newSubquerySelectAliasedNodes.get(0).getAlias(), null), !existsParseNode.isNegate()); tableNode = NODE_FACTORY.join(joinType, tableNode, subqueryDerivedTableNode, joinOnConditionParseNode, false); + recordSubqueryRewrite(existsParseNode.isNegate() ? "NOT EXISTS" : "EXISTS", joinType); return resultWhereParseNode; } @@ -411,7 +443,7 @@ public ParseNode visitLeave(ComparisonParseNode node, List l) throws SelectStatement subquery = fixSubqueryStatement(subqueryNode.getSelectNode()); String rhsTableAlias = ParseNodeFactory.createTempAlias(); JoinConditionExtractor conditionExtractor = - new JoinConditionExtractor(subquery, columnResolver, connection, rhsTableAlias); + new JoinConditionExtractor(subquery, columnResolver, connection, rhsTableAlias, context); ParseNode where = subquery.getWhere() == null ? null : subquery.getWhere().accept(conditionExtractor); if (where == subquery.getWhere()) { // non-correlated comparison subquery, add LIMIT 2, @@ -459,6 +491,7 @@ public ParseNode visitLeave(ComparisonParseNode node, List l) throws ParseNode ret = NODE_FACTORY.comparison(node.getFilterOp(), l.get(0), NODE_FACTORY .column(NODE_FACTORY.table(null, rhsTableAlias), selectNodes.get(0).getAlias(), null)); tableNode = NODE_FACTORY.join(joinType, tableNode, rhsTable, onNode, !isAggregate || isGroupby); + recordSubqueryRewrite("SCALAR", joinType); return ret; } @@ -497,7 +530,7 @@ protected List leaveArrayComparisonNode(ParseNode node, List createNewGroupByParseNodes( private static class JoinConditionExtractor extends AndRewriterBooleanParseNodeVisitor { private final TableName tableName; + private final String tableAlias; + private final StatementContext context; private ColumnResolveVisitor columnResolveVisitor; private List additionalSubselectSelectAliasedNodes; private List joinConditionParseNodes; public JoinConditionExtractor(SelectStatement subquery, ColumnResolver outerResolver, - PhoenixConnection connection, String tableAlias) throws SQLException { + PhoenixConnection connection, String tableAlias, StatementContext context) + throws SQLException { super(NODE_FACTORY); this.tableName = NODE_FACTORY.table(null, tableAlias); + this.tableAlias = tableAlias; + this.context = context; ColumnResolver localResolver = FromCompiler.getResolverForQuery(subquery, connection); this.columnResolveVisitor = new ColumnResolveVisitor(localResolver, outerResolver); this.additionalSubselectSelectAliasedNodes = Lists. newArrayList(); @@ -764,7 +802,11 @@ public ParseNode visitLeave(ComparisonParseNode node, List l) throws this.additionalSubselectSelectAliasedNodes .add(NODE_FACTORY.aliasedNode(alias, node.getLHS())); ParseNode lhsNode = NODE_FACTORY.column(tableName, alias, null); - this.joinConditionParseNodes.add(NODE_FACTORY.equal(lhsNode, node.getRHS())); + ParseNode joinCondition = NODE_FACTORY.equal(lhsNode, node.getRHS()); + if (context != null) { + context.putDecorrelatedSubqueryAlias(joinCondition, tableAlias); + } + this.joinConditionParseNodes.add(joinCondition); return null; } if ( @@ -775,7 +817,11 @@ public ParseNode visitLeave(ComparisonParseNode node, List l) throws this.additionalSubselectSelectAliasedNodes .add(NODE_FACTORY.aliasedNode(alias, node.getRHS())); ParseNode rhsNode = NODE_FACTORY.column(tableName, alias, null); - this.joinConditionParseNodes.add(NODE_FACTORY.equal(node.getLHS(), rhsNode)); + ParseNode joinCondition = NODE_FACTORY.equal(node.getLHS(), rhsNode); + if (context != null) { + context.putDecorrelatedSubqueryAlias(joinCondition, tableAlias); + } + this.joinConditionParseNodes.add(joinCondition); return null; } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java index 2282e2fd414..af561131250 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubselectRewriter.java @@ -298,8 +298,8 @@ public static SelectStatement pruneSelectAliasedNodes(SelectStatement selectStat newSelectAliasedNodes); } - public static SelectStatement flatten(SelectStatement select, PhoenixConnection connection) - throws SQLException { + public static SelectStatement flatten(SelectStatement select, PhoenixConnection connection, + StatementContext context) throws SQLException { TableNode from = select.getFrom(); while (from != null && from instanceof DerivedTableNode) { DerivedTableNode derivedTable = (DerivedTableNode) from; @@ -314,6 +314,9 @@ public static SelectStatement flatten(SelectStatement select, PhoenixConnection if (ret == select) { break; } + if (context != null) { + context.incrementDerivedTableFlattenCount(); + } select = ret; from = select.getFrom(); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UnionCompiler.java index b40de288b35..646d1cd5873 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UnionCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UnionCompiler.java @@ -118,6 +118,7 @@ public static TableRef contructSchemaTable(PhoenixStatement statement, List targetTypes) throws SQLException { PDataType type = expression.getDataType(); @@ -164,6 +165,7 @@ private static TupleProjector getTupleProjector(RowProjector rowProj, List convertToTupleProjectionPlan(List plans, Table * perform any special processing on the output of the subqueries. */ static void optimizeUnionOrderByIfPossible(UnionPlan innerUnionPlan, - SelectStatement outerSelectStatement, Supplier statementContextCreator) - throws SQLException { + SelectStatement outerSelectStatement, Supplier statementContextCreator, + StatementContext breadcrumbContext) throws SQLException { innerUnionPlan.enableCheckSupportOrderByOptimize(); if (!innerUnionPlan.isSupportOrderByOptimize()) { return; @@ -247,6 +249,11 @@ static void optimizeUnionOrderByIfPossible(UnionPlan innerUnionPlan, // subquery) could not compile out the outer query's group by or order by, we would // not perform any special processing on the output of the subqueries. innerUnionPlan.disableSupportOrderByOptimize(); + } else { + // The order-by merge optimization is preserved. Record a top-of-plan breadcrumb on the outer + // query's compilation context (the throwaway contexts produced by statementContextCreator are + // only used for trial group-by/order-by compilation and are discarded). + breadcrumbContext.addAppliedRewrite("UNION ORDER BY MERGE"); } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java index b9dfbc33e57..ad750d0e07b 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java @@ -62,6 +62,7 @@ import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.index.PhoenixIndexBuilderHelper; import org.apache.phoenix.index.PhoenixIndexCodec; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixResultSet; @@ -352,6 +353,7 @@ public UpsertCompiler(PhoenixStatement statement, Operation operation) { this.operation = operation; } + @SuppressWarnings("rawtypes") private static LiteralParseNode getNodeForRowTimestampColumn(PColumn col) { PDataType type = col.getDataType(); long dummyValue = 0L; @@ -579,14 +581,17 @@ public MutationPlan compile(UpsertStatement upsert) throws SQLException { if (valueNodesList.isEmpty()) { SelectStatement select = upsert.getSelect(); assert (select != null); - select = SubselectRewriter.flatten(select, connection); + // Pre-build a context so the early rewrite pass records top-of-plan breadcrumbs that are + // adopted by the UPSERT SELECT query plan's compilation context. + StatementContext rewriteContext = StatementContext.forRewrite(statement); + select = SubselectRewriter.flatten(select, connection, rewriteContext); ColumnResolver selectResolver = FromCompiler.getResolverForQuery(select, connection, false, upsert.getTable().getName()); select = StatementNormalizer.normalize(select, selectResolver); select = prependTenantAndViewConstants(table, select, tenantIdStr, addViewColumnsToBe, useServerTimestampToBe); SelectStatement transformedSelect = - SubqueryRewriter.transform(select, selectResolver, connection); + SubqueryRewriter.transform(select, selectResolver, connection, rewriteContext); if (transformedSelect != select) { selectResolver = FromCompiler.getResolverForQuery(transformedSelect, connection, false, upsert.getTable().getName()); @@ -649,7 +654,8 @@ public MutationPlan compile(UpsertStatement upsert) throws SQLException { // correctly // Use optimizer to choose the best plan QueryCompiler compiler = new QueryCompiler(statement, select, selectResolver, targetColumns, - parallelIteratorFactoryToBe, new SequenceManager(statement), true, false, null); + parallelIteratorFactoryToBe, new SequenceManager(statement), true, false, null) + .withRewriteContext(rewriteContext); queryPlanToBe = compiler.compile(); if (sameTable) { @@ -842,7 +848,8 @@ public MutationPlan compile(UpsertStatement upsert) throws SQLException { statementContext.getCurrentTable(), aggProjector, null, null, OrderBy.EMPTY_ORDER_BY, null, GroupBy.EMPTY_GROUP_BY, null, originalQueryPlan); return new ServerUpsertSelectMutationPlan(queryPlan, tableRef, originalQueryPlan, context, - connection, scan, aggPlan, aggProjector, maxSize, maxSizeBytes); + connection, scan, aggPlan, aggProjector, upsert.isReturningRow(), maxSize, + maxSizeBytes); } } //////////////////////////////////////////////////////////////////// @@ -850,7 +857,7 @@ public MutationPlan compile(UpsertStatement upsert) throws SQLException { ///////////////////////////////////////////////////////////////////// return new ClientUpsertSelectMutationPlan(queryPlan, tableRef, originalQueryPlan, parallelIteratorFactory, projector, columnIndexes, pkSlotIndexes, useServerTimestamp, - maxSize, maxSizeBytes); + upsert.isReturningRow(), maxSize, maxSizeBytes); } //////////////////////////////////////////////////////////////////// @@ -963,7 +970,8 @@ public MutationPlan compile(UpsertStatement upsert) throws SQLException { return new UpsertValuesMutationPlan(context, tableRef, nodeIndexOffset, constantExpressionsList, allColumns, columnIndexes, overlapViewColumns, valuesList, addViewColumns, connection, - pkSlotIndexes, useServerTimestamp, onDupKeyBytes, onDupKeyType, maxSize, maxSizeBytes); + pkSlotIndexes, useServerTimestamp, onDupKeyBytes, onDupKeyType, upsert.getOnDupKeyPairs(), + upsert.isReturningRow(), maxSize, maxSizeBytes); } private static byte[] getOnDuplicateKeyBytes(PTable table, StatementContext context, @@ -1148,12 +1156,14 @@ public class ServerUpsertSelectMutationPlan implements MutationPlan { private final Scan scan; private final QueryPlan aggPlan; private final RowProjector aggProjector; + private final boolean returningRow; private final int maxSize; private final long maxSizeBytes; public ServerUpsertSelectMutationPlan(QueryPlan queryPlan, TableRef tableRef, QueryPlan originalQueryPlan, StatementContext context, PhoenixConnection connection, - Scan scan, QueryPlan aggPlan, RowProjector aggProjector, int maxSize, long maxSizeBytes) { + Scan scan, QueryPlan aggPlan, RowProjector aggProjector, boolean returningRow, int maxSize, + long maxSizeBytes) { this.queryPlan = queryPlan; this.tableRef = tableRef; this.originalQueryPlan = originalQueryPlan; @@ -1162,6 +1172,7 @@ public ServerUpsertSelectMutationPlan(QueryPlan queryPlan, TableRef tableRef, this.scan = scan; this.aggPlan = aggPlan; this.aggProjector = aggProjector; + this.returningRow = returningRow; this.maxSize = maxSize; this.maxSizeBytes = maxSizeBytes; } @@ -1240,12 +1251,23 @@ public ExplainPlan getExplainPlan() throws SQLException { ExplainPlan explainPlan = aggPlan.getExplainPlan(); List queryPlanSteps = explainPlan.getPlanSteps(); ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - List planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 1); + List planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 2); ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); newBuilder.setAbstractExplainPlan("UPSERT ROWS"); + newBuilder.setReturningRow(returningRow); planSteps.add("UPSERT ROWS"); + if (returningRow) { + planSteps.add(" RETURNING *"); + } planSteps.addAll(queryPlanSteps); + // Surface the user's SELECT projection instead so VERBOSE explain describes the upsert. + ExplainTable.overrideMutationProject(planSteps, explainPlanAttributes, newBuilder, + queryPlan.getProjector()); + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); + } return new ExplainPlan(planSteps, newBuilder.build()); } @@ -1280,6 +1302,8 @@ private class UpsertValuesMutationPlan implements MutationPlan { private final boolean useServerTimestamp; private final byte[] onDupKeyBytes; private final OnDuplicateKeyType onDupKeyType; + private final List> onDupKeyPairs; + private final boolean returningRow; private final int maxSize; private final long maxSizeBytes; @@ -1288,7 +1312,8 @@ public UpsertValuesMutationPlan(StatementContext context, TableRef tableRef, int[] columnIndexes, Set overlapViewColumns, List valuesList, Set addViewColumns, PhoenixConnection connection, int[] pkSlotIndexes, boolean useServerTimestamp, byte[] onDupKeyBytes, OnDuplicateKeyType onDupKeyType, - int maxSize, long maxSizeBytes) { + List> onDupKeyPairs, boolean returningRow, int maxSize, + long maxSizeBytes) { this.context = context; this.tableRef = tableRef; this.nodeIndexOffset = nodeIndexOffset; @@ -1303,6 +1328,8 @@ public UpsertValuesMutationPlan(StatementContext context, TableRef tableRef, this.useServerTimestamp = useServerTimestamp; this.onDupKeyBytes = onDupKeyBytes; this.onDupKeyType = onDupKeyType; + this.onDupKeyPairs = onDupKeyPairs; + this.returningRow = returningRow; this.maxSize = maxSize; this.maxSizeBytes = maxSizeBytes; } @@ -1424,13 +1451,54 @@ public MutationState execute() throws SQLException { @Override public ExplainPlan getExplainPlan() throws SQLException { - List planSteps = Lists.newArrayListWithExpectedSize(2); + List planSteps = Lists.newArrayListWithExpectedSize(4); if (context.getSequenceManager().getSequenceCount() > 0) { planSteps .add("CLIENT RESERVE " + context.getSequenceManager().getSequenceCount() + " SEQUENCES"); } - planSteps.add("PUT SINGLE ROW"); - return new ExplainPlan(planSteps); + String header; + switch (onDupKeyType) { + case NONE: + header = "PUT SINGLE ROW"; + break; + case UPDATE: + header = "PUT SINGLE ROW ON DUPLICATE KEY UPDATE"; + break; + case UPDATE_ONLY: + header = "PUT SINGLE ROW ON DUPLICATE KEY UPDATE_ONLY"; + break; + case IGNORE: + header = "PUT SINGLE ROW ON DUPLICATE KEY IGNORE"; + break; + default: + throw new IllegalStateException("Unhandled OnDuplicateKeyType: " + onDupKeyType); + } + planSteps.add(header); + List serverUpdateSet = null; + if ( + onDupKeyType == OnDuplicateKeyType.UPDATE && onDupKeyPairs != null + && !onDupKeyPairs.isEmpty() + ) { + serverUpdateSet = Lists.newArrayListWithExpectedSize(onDupKeyPairs.size()); + for (Pair pair : onDupKeyPairs) { + serverUpdateSet.add(pair.getFirst().toString() + " = " + pair.getSecond().toString()); + } + planSteps.add(" SERVER UPDATE SET " + String.join(", ", serverUpdateSet)); + } + if (returningRow) { + planSteps.add(" RETURNING *"); + } + ExplainPlanAttributesBuilder builder = + new ExplainPlanAttributesBuilder(ExplainPlanAttributes.getDefaultExplainPlan()); + builder + .setOnDuplicateKeyAction(onDupKeyType == OnDuplicateKeyType.NONE ? null : onDupKeyType); + builder.setServerUpdateSet(serverUpdateSet); + builder.setReturningRow(returningRow); + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); + } + return new ExplainPlan(planSteps, builder.build()); } @Override @@ -1458,13 +1526,14 @@ private class ClientUpsertSelectMutationPlan implements MutationPlan { private final int[] columnIndexes; private final int[] pkSlotIndexes; private final boolean useServerTimestamp; + private final boolean returningRow; private final int maxSize; private final long maxSizeBytes; public ClientUpsertSelectMutationPlan(QueryPlan queryPlan, TableRef tableRef, QueryPlan originalQueryPlan, UpsertingParallelIteratorFactory parallelIteratorFactory, RowProjector projector, int[] columnIndexes, int[] pkSlotIndexes, boolean useServerTimestamp, - int maxSize, long maxSizeBytes) { + boolean returningRow, int maxSize, long maxSizeBytes) { this.queryPlan = queryPlan; this.tableRef = tableRef; this.originalQueryPlan = originalQueryPlan; @@ -1473,6 +1542,7 @@ public ClientUpsertSelectMutationPlan(QueryPlan queryPlan, TableRef tableRef, this.columnIndexes = columnIndexes; this.pkSlotIndexes = pkSlotIndexes; this.useServerTimestamp = useServerTimestamp; + this.returningRow = returningRow; this.maxSize = maxSize; this.maxSizeBytes = maxSizeBytes; queryPlan.getContext().setClientSideUpsertSelect(true); @@ -1548,12 +1618,20 @@ public ExplainPlan getExplainPlan() throws SQLException { ExplainPlan explainPlan = queryPlan.getExplainPlan(); List queryPlanSteps = explainPlan.getPlanSteps(); ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - List planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 1); + List planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 2); ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); newBuilder.setAbstractExplainPlan("UPSERT SELECT"); + newBuilder.setReturningRow(returningRow); planSteps.add("UPSERT SELECT"); + if (returningRow) { + planSteps.add(" RETURNING *"); + } planSteps.addAll(queryPlanSteps); + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); + } return new ExplainPlan(planSteps, newBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/WhereCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/WhereCompiler.java index 6dee0c2f514..4d4ac7717fe 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/WhereCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/WhereCompiler.java @@ -254,11 +254,32 @@ public static Expression compile(StatementContext context, FilterableStatement s scan.withStopRow(whereCompiler.getScanEndKey()); } } + // Tag the residual predicate(s) that become the server-side filter with their "WHERE" origin + // so EXPLAIN can attribute each SERVER FILTER BY line. + tagWhereOrigins(context, expression); + setScanFilter(context, statement, expression, whereCompiler.disambiguateWithFamily); return expression; } + /** + * Tag the top-level conjuncts of the residual WHERE expression with their origin for VERBOSE + * predicate source attribution. + */ + private static void tagWhereOrigins(StatementContext context, Expression expression) { + if (expression == null) { + return; + } + if (expression instanceof AndExpression) { + for (Expression child : expression.getChildren()) { + context.tagPredicate(child, "WHERE"); + } + } else { + context.tagPredicate(expression, "WHERE"); + } + } + public static class WhereExpressionCompiler extends ExpressionCompiler { protected boolean disambiguateWithFamily; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/AggregatePlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/AggregatePlan.java index 2c5d62b2a55..8a8ad87f8a9 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/AggregatePlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/AggregatePlan.java @@ -330,7 +330,7 @@ protected ResultIterator newIterator(ParallelScanGrouper scanGrouper, Scan scan, } if (having != null) { - aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having); + aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having, context); } if (statement.isDistinct() && statement.isAggregate()) { // Dedup on client if select distinct diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index e0dbc5afad9..08076542c4d 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -52,11 +52,13 @@ import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.iterate.DefaultParallelScanGrouper; import org.apache.phoenix.iterate.DelegateResultIterator; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.ParallelIteratorFactory; import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; +import org.apache.phoenix.optimize.OptimizerDecision; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.parse.ParseNodeFactory; @@ -110,6 +112,7 @@ public abstract class BaseQueryPlan implements QueryPlan { protected Long estimateInfoTimestamp; private boolean getEstimatesCalled; protected boolean isApplicable = true; + private OptimizerDecision optimizerDecision; protected BaseQueryPlan(StatementContext context, FilterableStatement statement, TableRef table, RowProjector projection, ParameterMetaData paramMetaData, Integer limit, Integer offset, @@ -539,10 +542,21 @@ private List getPlanSteps(ResultIterator iterator) { return planSteps; } - private Pair, ExplainPlanAttributes> getPlanStepsV2(ResultIterator iterator) { + private Pair, ExplainPlanAttributes> getPlanStepsV2(ResultIterator iterator) + throws SQLException { List planSteps = Lists.newArrayListWithExpectedSize(5); ExplainPlanAttributesBuilder builder = new ExplainPlanAttributesBuilder(); iterator.explain(planSteps, builder); + OptimizerDecision decision = getOptimizerDecision(); + if (decision != null) { + builder.setIndexRule(decision.getRule()); + builder.setFunctionalMatch(decision.getFunctionalMatch()); + builder.setIndexRejected(decision.getRejectedIndexes()); + } + if (context.isRoot()) { + ExplainTable.populateTopOfPlanAttributes(builder, context, getTableRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); + } return Pair.of(planSteps, builder.build()); } @@ -585,6 +599,16 @@ public void setApplicable(boolean isApplicable) { this.isApplicable = isApplicable; } + @Override + public OptimizerDecision getOptimizerDecision() { + return optimizerDecision; + } + + @Override + public void setOptimizerDecision(OptimizerDecision decision) { + this.optimizerDecision = decision; + } + private void getEstimates() throws SQLException { getEstimatesCalled = true; // Initialize a dummy iterator to get the estimates based on stats. diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java index e89bf515ccd..aef8ca24ab8 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.apache.hadoop.hbase.Cell; @@ -29,6 +30,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.GroupByCompiler.GroupBy; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; @@ -48,6 +50,7 @@ import org.apache.phoenix.iterate.BaseGroupedAggregatingResultIterator; import org.apache.phoenix.iterate.ClientHashAggregatingResultIterator; import org.apache.phoenix.iterate.DistinctAggregatingResultIterator; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.FilterAggregatingResultIterator; import org.apache.phoenix.iterate.FilterResultIterator; import org.apache.phoenix.iterate.GroupedAggregatingResultIterator; @@ -135,7 +138,7 @@ public Cost getCost() { public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { ResultIterator iterator = delegate.iterator(scanGrouper, scan); if (where != null) { - iterator = new FilterResultIterator(iterator, where); + iterator = new FilterResultIterator(iterator, where, context); } AggregatingResultIterator aggResultIterator; @@ -185,7 +188,7 @@ public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throw } if (having != null) { - aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having); + aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having, context); } if (statement.isDistinct() && statement.isAggregate()) { // Dedup on client if select distinct @@ -220,71 +223,112 @@ public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throw @Override public ExplainPlan getExplainPlan() throws SQLException { + // Carry the requested EXPLAIN options down to the delegate so every participating scan renders + // the same disclosures (e.g. VERBOSE predicate-origin attribution) as the driver scan. + delegate.getContext().setExplainOptions(context.getExplainOptions()); ExplainPlan explainPlan = delegate.getExplainPlan(); List planSteps = Lists.newArrayList(explainPlan.getPlanSteps()); ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); if (where != null) { - planSteps.add("CLIENT FILTER BY " + where.toString()); - newBuilder.setClientFilterBy(where.toString()); + if (context.isVerbose()) { + List filterLines = new ArrayList<>(); + List clientFilters = ExplainTable.renderVerboseFilters(context, where, + where.toString(), "CLIENT FILTER BY", filterLines); + for (String filterLine : filterLines) { + planSteps.add(filterLine); + newBuilder.addClientStep(filterLine); + } + newBuilder.setClientFilters(clientFilters); + } else { + String step = "CLIENT FILTER BY " + where.toString(); + planSteps.add(step); + newBuilder.setClientFilterBy(where.toString()); + newBuilder.addClientStep(step); + } } if (groupBy.isEmpty()) { - planSteps.add("CLIENT AGGREGATE INTO SINGLE ROW"); - newBuilder.setClientAggregate("CLIENT AGGREGATE INTO SINGLE ROW"); + String step = "CLIENT AGGREGATE INTO SINGLE ROW"; + planSteps.add(step); + newBuilder.setClientAggregate(step); + newBuilder.addClientStep(step); } else if (groupBy.isOrderPreserving()) { - planSteps.add( - "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY " + groupBy.getExpressions().toString()); - newBuilder.setClientAggregate( - "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY " + groupBy.getExpressions().toString()); + String step = + "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY " + groupBy.getExpressions().toString(); + planSteps.add(step); + newBuilder.setClientAggregate(step); + newBuilder.addClientStep(step); } else if (useHashAgg) { - planSteps - .add("CLIENT HASH AGGREGATE INTO DISTINCT ROWS BY " + groupBy.getExpressions().toString()); - newBuilder.setClientAggregate( - "CLIENT HASH AGGREGATE INTO DISTINCT ROWS BY " + groupBy.getExpressions().toString()); + String step = + "CLIENT HASH AGGREGATE INTO DISTINCT ROWS BY " + groupBy.getExpressions().toString(); + planSteps.add(step); + newBuilder.setClientAggregate(step); + newBuilder.addClientStep(step); if (orderBy == OrderBy.FWD_ROW_KEY_ORDER_BY || orderBy == OrderBy.REV_ROW_KEY_ORDER_BY) { - planSteps.add("CLIENT SORTED BY " + groupBy.getKeyExpressions().toString()); + String sortStep = "CLIENT SORTED BY " + groupBy.getKeyExpressions().toString(); + planSteps.add(sortStep); newBuilder.setClientSortedBy(groupBy.getKeyExpressions().toString()); + newBuilder.addClientStep(sortStep); } } else { - planSteps.add("CLIENT SORTED BY " + groupBy.getKeyExpressions().toString()); - planSteps - .add("CLIENT AGGREGATE INTO DISTINCT ROWS BY " + groupBy.getExpressions().toString()); + String sortStep = "CLIENT SORTED BY " + groupBy.getKeyExpressions().toString(); + String aggStep = + "CLIENT AGGREGATE INTO DISTINCT ROWS BY " + groupBy.getExpressions().toString(); + planSteps.add(sortStep); + planSteps.add(aggStep); newBuilder.setClientSortedBy(groupBy.getKeyExpressions().toString()); - newBuilder.setClientAggregate( - "CLIENT AGGREGATE INTO DISTINCT ROWS BY " + groupBy.getExpressions().toString()); + newBuilder.setClientAggregate(aggStep); + newBuilder.addClientStep(sortStep); + newBuilder.addClientStep(aggStep); } if (having != null) { - planSteps.add("CLIENT AFTER-AGGREGATION FILTER BY " + having.toString()); - newBuilder.setClientAfterAggregate("CLIENT AFTER-AGGREGATION FILTER BY " + having.toString()); + String step = "CLIENT AFTER-AGGREGATION FILTER BY " + having.toString(); + planSteps.add(step); + newBuilder.setClientAfterAggregate(step); + newBuilder.addClientStep(step); } if (statement.isDistinct() && statement.isAggregate()) { - planSteps.add("CLIENT DISTINCT ON " + projector.toString()); + String step = "CLIENT DISTINCT ON " + projector.toString(); + planSteps.add(step); newBuilder.setClientDistinctFilter(projector.toString()); + newBuilder.addClientStep(step); } if (offset != null) { - planSteps.add("CLIENT OFFSET " + offset); + String step = "CLIENT OFFSET " + offset; + planSteps.add(step); newBuilder.setClientOffset(offset); + newBuilder.addClientStep(step); } if (orderBy.getOrderByExpressions().isEmpty()) { if (limit != null) { - planSteps.add("CLIENT " + limit + " ROW LIMIT"); + String step = "CLIENT " + limit + " ROW LIMIT"; + planSteps.add(step); newBuilder.setClientRowLimit(limit); + newBuilder.addClientStep(step); } } else { - planSteps - .add("CLIENT" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S")) - + " SORTED BY " + orderBy.getOrderByExpressions().toString()); + String step = + "CLIENT" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S")) + + " SORTED BY " + orderBy.getOrderByExpressions().toString(); + planSteps.add(step); newBuilder.setClientRowLimit(limit); newBuilder.setClientSortedBy(orderBy.getOrderByExpressions().toString()); + newBuilder.addClientStep(step); } if (context.getSequenceManager().getSequenceCount() > 0) { int nSequences = context.getSequenceManager().getSequenceCount(); - planSteps.add( - "CLIENT RESERVE VALUES FROM " + nSequences + " SEQUENCE" + (nSequences == 1 ? "" : "S")); + String step = + "CLIENT RESERVE VALUES FROM " + nSequences + " SEQUENCE" + (nSequences == 1 ? "" : "S"); + planSteps.add(step); newBuilder.setClientSequenceCount(nSequences); + newBuilder.addClientStep(step); } + if (context.isRoot()) { + ExplainTable.populateTopOfPlanAttributes(newBuilder, context, getTableRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); + } return new ExplainPlan(planSteps, newBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java index 8bc71989604..06ea050f5f0 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java @@ -18,11 +18,13 @@ package org.apache.phoenix.execute; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.apache.hadoop.hbase.client.Scan; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.QueryPlan; @@ -31,6 +33,7 @@ import org.apache.phoenix.execute.visitor.ByteCountVisitor; import org.apache.phoenix.execute.visitor.QueryPlanVisitor; import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.FilterResultIterator; import org.apache.phoenix.iterate.LimitingResultIterator; import org.apache.phoenix.iterate.OffsetResultIterator; @@ -86,7 +89,7 @@ public T accept(QueryPlanVisitor visitor) { public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { ResultIterator iterator = delegate.iterator(scanGrouper, scan); if (where != null) { - iterator = new FilterResultIterator(iterator, where); + iterator = new FilterResultIterator(iterator, where, context); } if (!orderBy.getOrderByExpressions().isEmpty()) { // TopN @@ -116,6 +119,9 @@ public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throw @Override public ExplainPlan getExplainPlan() throws SQLException { + // Carry the requested EXPLAIN options down to the delegate so every participating scan renders + // the same disclosures (e.g. VERBOSE predicate-origin attribution) as the driver scan. + delegate.getContext().setExplainOptions(context.getExplainOptions()); ExplainPlan explainPlan = delegate.getExplainPlan(); List currentPlanSteps = explainPlan.getPlanSteps(); ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); @@ -123,36 +129,63 @@ public ExplainPlan getExplainPlan() throws SQLException { ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); if (where != null) { - planSteps.add("CLIENT FILTER BY " + where.toString()); - newBuilder.setClientFilterBy(where.toString()); + if (context.isVerbose()) { + List filterLines = new ArrayList<>(); + List clientFilters = ExplainTable.renderVerboseFilters(context, where, + where.toString(), "CLIENT FILTER BY", filterLines); + for (String filterLine : filterLines) { + planSteps.add(filterLine); + newBuilder.addClientStep(filterLine); + } + newBuilder.setClientFilters(clientFilters); + } else { + String step = "CLIENT FILTER BY " + where.toString(); + planSteps.add(step); + newBuilder.setClientFilterBy(where.toString()); + newBuilder.addClientStep(step); + } } if (!orderBy.getOrderByExpressions().isEmpty()) { if (offset != null) { - planSteps.add("CLIENT OFFSET " + offset); + String step = "CLIENT OFFSET " + offset; + planSteps.add(step); newBuilder.setClientOffset(offset); + newBuilder.addClientStep(step); } - planSteps - .add("CLIENT" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S")) - + " SORTED BY " + orderBy.getOrderByExpressions().toString()); + String step = + "CLIENT" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S")) + + " SORTED BY " + orderBy.getOrderByExpressions().toString(); + planSteps.add(step); newBuilder.setClientRowLimit(limit); newBuilder.setClientSortedBy(orderBy.getOrderByExpressions().toString()); + newBuilder.addClientStep(step); } else { if (offset != null) { - planSteps.add("CLIENT OFFSET " + offset); + String step = "CLIENT OFFSET " + offset; + planSteps.add(step); newBuilder.setClientOffset(offset); + newBuilder.addClientStep(step); } if (limit != null) { - planSteps.add("CLIENT " + limit + " ROW LIMIT"); + String step = "CLIENT " + limit + " ROW LIMIT"; + planSteps.add(step); newBuilder.setClientRowLimit(limit); + newBuilder.addClientStep(step); } } if (context.getSequenceManager().getSequenceCount() > 0) { int nSequences = context.getSequenceManager().getSequenceCount(); - planSteps.add( - "CLIENT RESERVE VALUES FROM " + nSequences + " SEQUENCE" + (nSequences == 1 ? "" : "S")); + String step = + "CLIENT RESERVE VALUES FROM " + nSequences + " SEQUENCE" + (nSequences == 1 ? "" : "S"); + planSteps.add(step); newBuilder.setClientSequenceCount(nSequences); + newBuilder.addClientStep(step); } + if (context.isRoot()) { + ExplainTable.populateTopOfPlanAttributes(newBuilder, context, getTableRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); + } return new ExplainPlan(planSteps, newBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java index b78a4ab6144..ac3616e2c0c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java @@ -32,12 +32,17 @@ import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; import org.apache.phoenix.optimize.Cost; +import org.apache.phoenix.optimize.OptimizerDecision; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.TableRef; public abstract class DelegateQueryPlan implements QueryPlan { protected final QueryPlan delegate; + // Fallback storage for the optimizer decision used only when this plan has no delegate to + // forward to. Some DelegateQueryPlan subclasses intentionally pass a null delegate and override + // getContext()/getTableRef()/getProjector() to serve from local state. + private OptimizerDecision optimizerDecision; public DelegateQueryPlan(QueryPlan delegate) { this.delegate = delegate; @@ -171,4 +176,18 @@ public List getOutputOrderBys() { public boolean isApplicable() { return delegate.isApplicable(); } + + @Override + public OptimizerDecision getOptimizerDecision() { + return delegate != null ? delegate.getOptimizerDecision() : optimizerDecision; + } + + @Override + public void setOptimizerDecision(OptimizerDecision decision) { + if (delegate != null) { + delegate.setOptimizerDecision(decision); + } else { + this.optimizerDecision = decision; + } + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java index 81500776dbb..c48b67276a9 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java @@ -40,7 +40,10 @@ import org.apache.phoenix.cache.ServerCacheClient.ServerCache; import org.apache.phoenix.compile.ColumnProjector; import org.apache.phoenix.compile.ExplainPlan; +import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.FromCompiler; +import org.apache.phoenix.compile.JoinCompiler; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.compile.RowProjector; import org.apache.phoenix.compile.ScanRanges; @@ -97,6 +100,7 @@ public class HashJoinPlan extends DelegateQueryPlan { private final SelectStatement statement; private final HashJoinInfo joinInfo; + private JoinCompiler.Strategy strategy; private final SubPlan[] subPlans; private final boolean recompileWhereClause; private final Set tableRefs; @@ -113,9 +117,10 @@ public class HashJoinPlan extends DelegateQueryPlan { private boolean hasSubPlansWithPersistentCache; public static HashJoinPlan create(SelectStatement statement, QueryPlan plan, - HashJoinInfo joinInfo, SubPlan[] subPlans) throws SQLException { - if (!(plan instanceof HashJoinPlan)) return new HashJoinPlan(statement, plan, joinInfo, - subPlans, joinInfo == null, Collections. emptyMap()); + HashJoinInfo joinInfo, SubPlan[] subPlans, JoinCompiler.Strategy strategy) throws SQLException { + if (!(plan instanceof HashJoinPlan)) + return new HashJoinPlan(statement, plan, joinInfo, subPlans, joinInfo == null, + Collections. emptyMap(), strategy); HashJoinPlan hashJoinPlan = (HashJoinPlan) plan; assert (hashJoinPlan.joinInfo == null && hashJoinPlan.delegate instanceof BaseQueryPlan); @@ -128,16 +133,18 @@ public static HashJoinPlan create(SelectStatement statement, QueryPlan plan, mergedSubPlans[i++] = subPlan; } return new HashJoinPlan(statement, hashJoinPlan.delegate, joinInfo, mergedSubPlans, true, - hashJoinPlan.dependencies); + hashJoinPlan.dependencies, strategy); } private HashJoinPlan(SelectStatement statement, QueryPlan plan, HashJoinInfo joinInfo, SubPlan[] subPlans, boolean recompileWhereClause, - Map dependencies) throws SQLException { + Map dependencies, JoinCompiler.Strategy strategy) + throws SQLException { super(plan); this.dependencies.putAll(dependencies); this.statement = statement; this.joinInfo = joinInfo; + this.strategy = strategy; this.subPlans = subPlans; this.recompileWhereClause = recompileWhereClause; this.tableRefs = Sets.newHashSetWithExpectedSize(subPlans.length + plan.getSourceRefs().size()); @@ -266,7 +273,7 @@ public TaskExecutionMetricsHolder getTaskExecutionMetric() { ? delegate.iterator(scanGrouper, scan) : ((BaseQueryPlan) delegate).iterator(dependencies, scanGrouper, scan); if (statement.getInnerSelectStatement() != null && postFilter != null) { - iterator = new FilterResultIterator(iterator, postFilter); + iterator = new FilterResultIterator(iterator, postFilter, delegate.getContext()); } if (hasSubPlansWithPersistentCache) { @@ -313,24 +320,71 @@ private Expression createKeyRangeExpression(Expression lhsExpression, Expression @Override public ExplainPlan getExplainPlan() throws SQLException { - // TODO : Support ExplainPlanAttributes for HashJoinPlan - List planSteps = Lists.newArrayList(delegate.getExplainPlan().getPlanSteps()); + ExplainPlan delegateExplainPlan = delegate.getExplainPlan(); + List planSteps = Lists.newArrayList(delegateExplainPlan.getPlanSteps()); + // Each hash/skip-scan sub-plan is recorded under subPlans. + ExplainPlanAttributes delegateAttributes = delegateExplainPlan.getPlanStepsAsAttributes(); + if (delegateAttributes == null) { + delegateAttributes = ExplainPlanAttributes.getDefaultExplainPlan(); + } + ExplainPlanAttributesBuilder builder = new ExplainPlanAttributesBuilder(delegateAttributes); + List subPlanAttributes = Lists.newArrayList(); int count = subPlans.length; + // Carry the requested EXPLAIN options down to each sub-plan so every participating scan renders + // the same disclosures (e.g. VERBOSE predicate-origin attribution) as the driver scan. + for (int i = 0; i < count; i++) { + subPlans[i].getInnerPlan().getContext().setExplainOptions(getContext().getExplainOptions()); + } for (int i = 0; i < count; i++) { planSteps.addAll(subPlans[i].getPreSteps(this)); + ExplainPlanAttributes subPlanAttribute = subPlans[i].getPreStepsAsAttributes(this); + if (subPlanAttribute != null) { + subPlanAttributes.add(subPlanAttribute); + } } for (int i = 0; i < count; i++) { - planSteps.addAll(subPlans[i].getPostSteps(this)); + List postSteps = subPlans[i].getPostSteps(this); + planSteps.addAll(postSteps); + for (String step : postSteps) { + String trimmed = step.trim(); + if (trimmed.startsWith("DYNAMIC SERVER FILTER BY")) { + builder.setDynamicServerFilter(trimmed); + } + } } if (joinInfo != null && joinInfo.getPostJoinFilterExpression() != null) { - planSteps.add( - " AFTER-JOIN SERVER FILTER BY " + joinInfo.getPostJoinFilterExpression().toString()); + String afterJoinFilter = + "AFTER-JOIN SERVER FILTER BY " + joinInfo.getPostJoinFilterExpression().toString(); + planSteps.add(" " + afterJoinFilter); + builder.setAfterJoinFilter(afterJoinFilter); } if (joinInfo != null && joinInfo.getLimit() != null) { planSteps.add(" JOIN-SCANNER " + joinInfo.getLimit() + " ROW LIMIT"); + builder.setJoinScannerLimit(joinInfo.getLimit().longValue()); + } + if (!subPlanAttributes.isEmpty()) { + builder.setSubPlans(subPlanAttributes); + } + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTableRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); } - return new ExplainPlan(planSteps); + return new ExplainPlan(planSteps, builder.build()); + } + + /** + * Builds the explain-plan attributes for a hash-join subplan by taking the inner plan's + * attributes (when available) and stamping the supplied join header onto abstractExplainPlan. + */ + private static ExplainPlanAttributes subPlanAttributesWithHeader(QueryPlan plan, String header) + throws SQLException { + ExplainPlanAttributes innerAttributes = plan.getExplainPlan().getPlanStepsAsAttributes(); + ExplainPlanAttributesBuilder builder = + (innerAttributes != null && innerAttributes != ExplainPlanAttributes.getDefaultExplainPlan()) + ? new ExplainPlanAttributesBuilder(innerAttributes) + : new ExplainPlanAttributesBuilder(); + return builder.setAbstractExplainPlan(header).build(); } @Override @@ -342,6 +396,14 @@ public HashJoinInfo getJoinInfo() { return joinInfo; } + public JoinCompiler.Strategy getStrategy() { + return strategy; + } + + public void setStrategy(JoinCompiler.Strategy strategy) { + this.strategy = strategy; + } + public SubPlan[] getSubPlans() { return subPlans; } @@ -430,6 +492,14 @@ public interface SubPlan { public List getPostSteps(HashJoinPlan parent) throws SQLException; + /** + * Returns the explain-plan attributes for this subplan, with its abstractExplainPlan. Returns + * null when no structured attributes are available for the inner plan. + */ + default ExplainPlanAttributes getPreStepsAsAttributes(HashJoinPlan parent) throws SQLException { + return null; + } + public QueryPlan getInnerPlan(); public boolean hasKeyRangeExpression(); @@ -446,6 +516,7 @@ public WhereClauseSubPlan(QueryPlan plan, SelectStatement select, boolean expect this.expectSingleRow = expectSingleRow; } + @SuppressWarnings("rawtypes") @Override public ServerCache execute(HashJoinPlan parent) throws SQLException { List values = Lists. newArrayList(); @@ -508,6 +579,12 @@ public List getPreSteps(HashJoinPlan parent) throws SQLException { return steps; } + @Override + public ExplainPlanAttributes getPreStepsAsAttributes(HashJoinPlan parent) throws SQLException { + String header = "EXECUTE " + (expectSingleRow ? "SINGLE" : "MULTIPLE") + "-ROW SUBQUERY"; + return subPlanAttributesWithHeader(plan, header); + } + @Override public List getPostSteps(HashJoinPlan parent) throws SQLException { return Collections. emptyList(); @@ -628,24 +705,52 @@ public void postProcess(ServerCache result, HashJoinPlan parent) throws SQLExcep } } + /** + * Builds the join operator line (without leading indentation) for this subplan, with the chosen + * {@link JoinCompiler.Strategy} and the {@code SKIP MERGE} / {@code DELAYED EVALUATION} + * decorators rendered as a single trailing SQL comment. + */ + private String buildHeader(HashJoinPlan parent) { + boolean isHashJoin = hashExpressions != null; + String op = isHashJoin + ? "PARALLEL " + parent.joinInfo.getJoinTypes()[index].toString().toUpperCase() + + "-JOIN TABLE " + index + : "SKIP-SCAN-JOIN TABLE " + index; + JoinCompiler.Strategy strategy = parent.getStrategy(); + if (strategy == null) { + return op; + } + StringBuilder comment = new StringBuilder("/* HASH BUILD "); + comment.append(strategy == JoinCompiler.Strategy.HASH_BUILD_LEFT ? "LEFT" : "RIGHT"); + if (isHashJoin) { + boolean skipMerge = parent.joinInfo.getSchemas()[index].getFieldCount() == 0; + boolean earlyEvaluation = parent.joinInfo.earlyEvaluation()[index]; + if (skipMerge) { + comment.append(", SKIP MERGE"); + } + if (!earlyEvaluation) { + comment.append(", DELAYED EVALUATION"); + } + } + comment.append(" */"); + return op + " " + comment; + } + @Override public List getPreSteps(HashJoinPlan parent) throws SQLException { List steps = Lists.newArrayList(); - boolean earlyEvaluation = parent.joinInfo.earlyEvaluation()[index]; - boolean skipMerge = parent.joinInfo.getSchemas()[index].getFieldCount() == 0; - if (hashExpressions != null) { - steps.add(" PARALLEL " + parent.joinInfo.getJoinTypes()[index].toString().toUpperCase() - + "-JOIN TABLE " + index + (earlyEvaluation ? "" : "(DELAYED EVALUATION)") - + (skipMerge ? " (SKIP MERGE)" : "")); - } else { - steps.add(" SKIP-SCAN-JOIN TABLE " + index); - } + steps.add(" " + buildHeader(parent)); for (String step : plan.getExplainPlan().getPlanSteps()) { steps.add(" " + step); } return steps; } + @Override + public ExplainPlanAttributes getPreStepsAsAttributes(HashJoinPlan parent) throws SQLException { + return subPlanAttributesWithHeader(plan, buildHeader(parent)); + } + @Override public List getPostSteps(HashJoinPlan parent) throws SQLException { if (keyRangeLhsExpression == null) return Collections. emptyList(); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java index 0b851902211..f0ae604516b 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java @@ -41,6 +41,7 @@ import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.GroupByCompiler.GroupBy; +import org.apache.phoenix.compile.JoinCompiler; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.QueryCompiler; import org.apache.phoenix.compile.QueryPlan; @@ -56,6 +57,7 @@ import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.OrderByExpression; import org.apache.phoenix.iterate.DefaultParallelScanGrouper; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.PhoenixQueues; import org.apache.phoenix.iterate.ResultIterator; @@ -98,6 +100,7 @@ public class SortMergeJoinPlan implements QueryPlan { * {@link JoinType#Left}. */ private final JoinType joinType; + private JoinCompiler.Strategy strategy = JoinCompiler.Strategy.SORT_MERGE; private final QueryPlan lhsPlan; private final QueryPlan rhsPlan; private final List lhsKeyExpressions; @@ -188,14 +191,15 @@ public ResultIterator iterator() throws SQLException { @Override public ExplainPlan getExplainPlan() throws SQLException { List steps = Lists.newArrayList(); - steps.add("SORT-MERGE-JOIN (" + joinType.toString().toUpperCase() + ") TABLES"); + steps + .add("SORT-MERGE-JOIN (" + joinType.toString().toUpperCase() + ") TABLES /* SORT_MERGE */"); + // Carry the requested EXPLAIN options down to both join halves so every participating scan + // renders the same disclosures (e.g. VERBOSE predicate-origin attribution) as the driver scan. + lhsPlan.getContext().setExplainOptions(getContext().getExplainOptions()); + rhsPlan.getContext().setExplainOptions(getContext().getExplainOptions()); ExplainPlan lhsExplainPlan = lhsPlan.getExplainPlan(); List lhsPlanSteps = lhsExplainPlan.getPlanSteps(); ExplainPlanAttributes lhsPlanAttributes = lhsExplainPlan.getPlanStepsAsAttributes(); - ExplainPlanAttributesBuilder lhsPlanBuilder = - new ExplainPlanAttributesBuilder(lhsPlanAttributes); - lhsPlanBuilder - .setAbstractExplainPlan("SORT-MERGE-JOIN (" + joinType.toString().toUpperCase() + ")"); for (String step : lhsPlanSteps) { steps.add(" " + step); @@ -205,15 +209,24 @@ public ExplainPlan getExplainPlan() throws SQLException { ExplainPlan rhsExplainPlan = rhsPlan.getExplainPlan(); List rhsPlanSteps = rhsExplainPlan.getPlanSteps(); ExplainPlanAttributes rhsPlanAttributes = rhsExplainPlan.getPlanStepsAsAttributes(); - ExplainPlanAttributesBuilder rhsPlanBuilder = - new ExplainPlanAttributesBuilder(rhsPlanAttributes); - - lhsPlanBuilder.setRhsJoinQueryExplainPlan(rhsPlanBuilder.build()); for (String step : rhsPlanSteps) { steps.add(" " + step); } - return new ExplainPlan(steps, lhsPlanBuilder.build()); + + // Build a synthetic root that holds the join operator and its two operands as separate + // child plans so nested sort-merge-joins can be represented. + ExplainPlanAttributesBuilder rootBuilder = new ExplainPlanAttributesBuilder(); + rootBuilder + .setAbstractExplainPlan("SORT-MERGE-JOIN (" + joinType.toString().toUpperCase() + ")"); + rootBuilder.setSortMergeSkipMerge(rhsSchema.getFieldCount() == 0); + rootBuilder.setLhsJoinQueryExplainPlan(lhsPlanAttributes); + rootBuilder.setRhsJoinQueryExplainPlan(rhsPlanAttributes); + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(rootBuilder, getContext(), getTableRef()); + ExplainTable.populateTopOfPlanEstimates(rootBuilder, this); + } + return new ExplainPlan(steps, rootBuilder.build()); } @Override @@ -302,6 +315,14 @@ public JoinType getJoinType() { return joinType; } + public JoinCompiler.Strategy getStrategy() { + return strategy; + } + + public void setStrategy(JoinCompiler.Strategy strategy) { + this.strategy = strategy; + } + private static SQLException closeIterators(ResultIterator lhsIterator, ResultIterator rhsIterator) { SQLException e = null; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/TupleProjectionPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/TupleProjectionPlan.java index ac38657f181..43e565b64ae 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/TupleProjectionPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/TupleProjectionPlan.java @@ -27,6 +27,7 @@ import org.apache.phoenix.compile.ColumnResolver; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.GroupByCompiler.GroupBy; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; @@ -40,6 +41,7 @@ import org.apache.phoenix.expression.OrderByExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.iterate.DelegateResultIterator; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.FilterResultIterator; import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; @@ -53,6 +55,7 @@ public class TupleProjectionPlan extends DelegateQueryPlan { private final TupleProjector tupleProjector; private final Expression postFilter; private final ColumnResolver columnResolver; + private final StatementContext statementContext; private final List actualOutputOrderBys; public TupleProjectionPlan(QueryPlan plan, TupleProjector tupleProjector, @@ -63,6 +66,7 @@ public TupleProjectionPlan(QueryPlan plan, TupleProjector tupleProjector, } this.tupleProjector = tupleProjector; this.postFilter = postFilter; + this.statementContext = statementContext; if (statementContext != null) { this.columnResolver = statementContext.getResolver(); this.actualOutputOrderBys = this.convertInputOrderBys(plan); @@ -143,14 +147,33 @@ private ProjectedColumnExpression getProjectedValueColumnExpression(int columnIn @Override public ExplainPlan getExplainPlan() throws SQLException { + // getContext() is the delegate's context (already carrying the requested EXPLAIN options). + // Carry those same options onto the separate post-filter context so this plan's CLIENT FILTER + // BY + // renders the same disclosures (e.g. VERBOSE predicate-origin attribution) as the driver scan. + if (statementContext != null) { + statementContext.setExplainOptions(getContext().getExplainOptions()); + } ExplainPlan explainPlan = delegate.getExplainPlan(); List planSteps = Lists.newArrayList(explainPlan.getPlanSteps()); ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); if (postFilter != null) { - planSteps.add("CLIENT FILTER BY " + postFilter.toString()); ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); - newBuilder.setClientFilterBy(postFilter.toString()); + if (statementContext != null && statementContext.isVerbose()) { + int from = planSteps.size(); + List filters = ExplainTable.renderVerboseFilters(statementContext, + postFilter, postFilter.toString(), "CLIENT FILTER BY", planSteps); + for (int i = from; i < planSteps.size(); i++) { + newBuilder.addClientStep(planSteps.get(i)); + } + newBuilder.setClientFilters(filters); + } else { + String step = "CLIENT FILTER BY " + postFilter.toString(); + planSteps.add(step); + newBuilder.setClientFilterBy(postFilter.toString()); + newBuilder.addClientStep(step); + } explainPlanAttributes = newBuilder.build(); } @@ -176,7 +199,7 @@ public String toString() { }; if (postFilter != null) { - iterator = new FilterResultIterator(iterator, postFilter); + iterator = new FilterResultIterator(iterator, postFilter, statementContext); } return iterator; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/UnionPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/UnionPlan.java index 1a1e76f6904..2350bfdd53d 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/UnionPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/UnionPlan.java @@ -40,6 +40,7 @@ import org.apache.phoenix.expression.OrderByExpression; import org.apache.phoenix.iterate.ConcatResultIterator; import org.apache.phoenix.iterate.DefaultParallelScanGrouper; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.LimitingResultIterator; import org.apache.phoenix.iterate.MergeSortTopNResultIterator; import org.apache.phoenix.iterate.OffsetResultIterator; @@ -238,18 +239,61 @@ public ExplainPlan getExplainPlan() throws SQLException { String abstractExplainPlan = "UNION ALL OVER " + this.plans.size() + " QUERIES"; builder.setAbstractExplainPlan(abstractExplainPlan); steps.add(abstractExplainPlan); - ResultIterator iterator = iterator(); - iterator.explain(steps, builder); - // Indent plans steps nested under union, except last client-side merge/concat step (if there is - // one) - int offset = - !orderBy.getOrderByExpressions().isEmpty() && limit != null ? 2 : limit != null ? 1 : 0; - for (int i = 1; i < steps.size() - offset; i++) { - steps.set(i, " " + steps.get(i)); + // Carry the requested EXPLAIN options down to each branch so every participating scan renders + // the same disclosures (e.g. VERBOSE predicate-origin attribution) as the driver scan. + for (QueryPlan plan : plans) { + plan.getContext().setExplainOptions(getContext().getExplainOptions()); + } + // Compose each branch from its own getExplainPlan() so the full sub-plan structure of every + // branch is preserved and explaining the union does not trigger sub-plan execution. + UnionResultIterators.explainBranches(plans, steps, builder); + addUnionTailLines(steps, builder); + if (getContext().isRoot()) { + ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTableRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); } return new ExplainPlan(steps, builder.build()); } + /** + * Appends the client-side merge/offset/limit lines that follow the union branches, mirroring the + * iterator chain assembled in {@link #iterator(ParallelScanGrouper, Scan)}. These lines stay at + * column 0, after the indented branch steps. + */ + private void addUnionTailLines(List steps, ExplainPlanAttributesBuilder builder) { + if (!orderBy.isEmpty() || this.supportOrderByOptimize) { + String mergeSort = "CLIENT MERGE SORT"; + steps.add(mergeSort); + builder.setClientSortAlgo(mergeSort); + builder.addClientStep(mergeSort); + if (offset != null && offset > 0) { + String step = "CLIENT OFFSET " + offset; + steps.add(step); + builder.setClientOffset(offset); + builder.addClientStep(step); + } + if (limit != null && limit > 0) { + String step = "CLIENT LIMIT " + limit; + steps.add(step); + builder.setClientRowLimit(limit); + builder.addClientStep(step); + } + } else { + if (offset != null) { + String step = "CLIENT OFFSET " + offset; + steps.add(step); + builder.setClientOffset(offset); + builder.addClientStep(step); + } + if (limit != null) { + String step = "CLIENT " + limit + " ROW LIMIT"; + steps.add(step); + builder.setClientRowLimit(limit); + builder.addClientStep(step); + } + } + } + @Override public long getEstimatedSize() { return DEFAULT_ESTIMATED_SIZE; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/LiteralExpression.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/LiteralExpression.java index 7075661567b..7735d121246 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/LiteralExpression.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/LiteralExpression.java @@ -265,10 +265,12 @@ public Determinism getDeterminism() { @Override public String toString() { - if (value == null && byteValue != null) { - return Bytes.toStringBinary(byteValue); - } else if (value == null) { - return "null"; + // Typed null literals carry an empty byteValue (ByteUtil.EMPTY_BYTE_ARRAY); render those as + // "null" rather than the empty string Bytes.toStringBinary would produce, so callers like + // FunctionExpression.toString that emit comma-separated children don't print a phantom + // trailing empty argument such as BSON_VALUE(payload, 'k', 'VARCHAR', ). + if (value == null) { + return (byteValue != null && byteValue.length > 0) ? Bytes.toStringBinary(byteValue) : "null"; } // TODO: move into PDataType? if (type.isCoercibleTo(PTimestamp.INSTANCE)) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 45e160fac8a..64ce3cb73e4 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -86,6 +86,7 @@ import org.apache.phoenix.hbase.index.util.VersionUtil; import org.apache.phoenix.join.HashCacheClient; import org.apache.phoenix.monitoring.OverAllQueryMetrics; +import org.apache.phoenix.optimize.OptimizerDecision; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; @@ -640,6 +641,21 @@ public List getSplits() { else return splits; } + @Override + protected int getSplitCount() { + return splits == null ? 0 : splits.size(); + } + + @Override + protected OptimizerDecision getOptimizerDecision() { + return plan.getOptimizerDecision(); + } + + @Override + protected org.apache.phoenix.compile.RowProjector getProjector() { + return plan.getProjector(); + } + @Override public List> getScans() { if (scans == null) return Collections.emptyList(); @@ -1839,8 +1855,8 @@ private void explainUtil(List planSteps, buf.append(estimatedRows).append(" ROWS "); buf.append(estimatedSize).append(" BYTES "); if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder.setEstimatedRows(estimatedRows); - explainPlanAttributesBuilder.setEstimatedSizeInBytes(estimatedSize); + explainPlanAttributesBuilder.setScanEstimatedRows(estimatedRows); + explainPlanAttributesBuilder.setScanEstimatedSizeInBytes(estimatedSize); } } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/CursorResultIterator.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/CursorResultIterator.java index 9c44579366b..113c6fad459 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/CursorResultIterator.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/CursorResultIterator.java @@ -60,7 +60,9 @@ public void explain(List planSteps, ExplainPlanAttributesBuilder explainPlanAttributesBuilder) { delegate.explain(planSteps, explainPlanAttributesBuilder); explainPlanAttributesBuilder.setClientCursorName(cursorName); - planSteps.add("CLIENT CURSOR " + cursorName); + String step = "CLIENT CURSOR " + cursorName; + planSteps.add(step); + explainPlanAttributesBuilder.addClientStep(step); } @Override diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/DistinctAggregatingResultIterator.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/DistinctAggregatingResultIterator.java index 10f081c8a48..6dfdfb8857e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/DistinctAggregatingResultIterator.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/DistinctAggregatingResultIterator.java @@ -134,7 +134,9 @@ public void explain(List planSteps, ExplainPlanAttributesBuilder explainPlanAttributesBuilder) { targetAggregatingResultIterator.explain(planSteps, explainPlanAttributesBuilder); explainPlanAttributesBuilder.setClientDistinctFilter(rowProjector.toString()); - planSteps.add("CLIENT DISTINCT ON " + rowProjector.toString()); + String step = "CLIENT DISTINCT ON " + rowProjector.toString(); + planSteps.add(step); + explainPlanAttributesBuilder.addClientStep(step); } @Override diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index 221a69c8dd8..ac6d927dd66 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -17,14 +17,15 @@ */ package org.apache.phoenix.iterate; +import java.sql.SQLException; import java.text.Format; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; -import java.util.NoSuchElementException; +import java.util.Map; import java.util.Set; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Consistency; @@ -35,15 +36,27 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.ColumnProjector; +import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.GroupByCompiler.GroupBy; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; +import org.apache.phoenix.compile.RowProjector; import org.apache.phoenix.compile.ScanRanges; import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.compile.StatementPlan; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.expression.AndExpression; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.function.BsonConditionExpressionFunction; +import org.apache.phoenix.expression.function.JsonExistsFunction; import org.apache.phoenix.filter.BooleanExpressionFilter; import org.apache.phoenix.filter.DistinctPrefixFilter; import org.apache.phoenix.filter.EmptyColumnOnlyFilter; +import org.apache.phoenix.optimize.OptimizerDecision; +import org.apache.phoenix.optimize.OptimizerReasons; +import org.apache.phoenix.optimize.RejectedIndexEntry; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.query.KeyRange; @@ -53,20 +66,18 @@ import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PTable; -import org.apache.phoenix.schema.RowKeySchema; +import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.util.CDCUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.StringUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public abstract class ExplainTable { - private static final Logger LOGGER = LoggerFactory.getLogger(ExplainTable.class); private static final List EVERYTHING = Collections.singletonList(KeyRange.EVERYTHING_RANGE); public static final String POINT_LOOKUP_ON_STRING = "POINT LOOKUP ON "; @@ -124,12 +135,179 @@ private String explainSkipScan() { return buf.toString(); } + /** + * Number of region scan splits the plan will hit, used to render the {@code REGIONS PLANNED} + * per-scan line. + * @return the split count, or 0 when unknown + */ + protected int getSplitCount() { + return 0; + } + + /** + * The optimizer's index selection rationale for the plan this scan belongs to, used to render the + * per-scan {@code INDEX} rule comment and the {@code !INDEX} rejection comments. Returns + * {@code null} when the plan did not participate in optimizer index selection (DML, DDL, and + * non-optimizer plans). + * @return the decision, or {@code null} when unavailable + */ + protected OptimizerDecision getOptimizerDecision() { + return null; + } + + /** + * The post-compile row projector for the plan this scan belongs to. Returns {@code null} when the + * plan has no projector. + */ + protected RowProjector getProjector() { + return null; + } + + /** + * Whether {@code rule} is a default rule whose {@code INDEX} comment is suppressed. The default + * rules are {@link OptimizerReasons#RULE_DATA_TABLE} (no candidate indexes considered) and + * {@link OptimizerReasons#RULE_ONLY_CANDIDATE} (a single viable candidate). + */ + private static boolean isDefaultRule(String rule) { + return OptimizerReasons.RULE_DATA_TABLE.equals(rule) + || OptimizerReasons.RULE_ONLY_CANDIDATE.equals(rule); + } + + /** + * Logical name used to render a table or index in EXPLAIN output. Shared by both the scan + * {@code OVER} line's local index decoration and the per scan {@code INDEX} line. + * @param table the scanned table or index + * @return the display name with any child-view local-index prefix stripped + */ + private static String getExplainIndexName(PTable table) { + String indexName = table.getName().getString(); + if ( + table.getIndexType() == PTable.IndexType.LOCAL && table.getViewIndexId() != null + && indexName.contains(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR) + ) { + int lastIndexOf = indexName.lastIndexOf(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR); + indexName = indexName.substring(lastIndexOf + 1); + } + return indexName; + } + + /** + * Populate the top-of-plan disclosure attributes on the supplied builder. Should be invoked only + * for a root plan (i.e. when {@code context.isRoot()}). + * @param builder the attributes builder to populate (no-op when null) + * @param context the root statement context + * @param tableRef the plan's primary table reference (may be null) + */ + public static void populateTopOfPlanAttributes(ExplainPlanAttributesBuilder builder, + StatementContext context, TableRef tableRef) { + if (builder == null || context == null) { + return; + } + if (context.getConnection() != null && context.getConnection().getTenantId() != null) { + builder.setTenantId(context.getConnection().getTenantId().getString()); + } + PTable table = tableRef == null ? null : tableRef.getTable(); + if (table != null) { + if (table.getType() == PTableType.VIEW) { + builder.setViewName(table.getName().getString()); + if (table.getBaseTableLogicalName() != null) { + builder.setViewBaseName(table.getBaseTableLogicalName().getString()); + } + } + if (table.isTransactional() && table.getTransactionProvider() != null) { + builder.setTxnProvider(table.getTransactionProvider().name()); + } + } + String cdcScopes = context.getEncodedCdcIncludeScopes(); + if (cdcScopes == null && table != null && table.getType() == PTableType.CDC) { + Set scopes = table.getCDCIncludeScopes(); + if (scopes != null && !scopes.isEmpty()) { + cdcScopes = CDCUtil.makeChangeScopeStringFromEnums(scopes); + } + } + if (cdcScopes != null) { + builder.setCdcScopes(cdcScopes); + } + // Aggregate rewrite breadcrumbs across the context and its sub-contexts, deduping while + // preserving first occurrence order. The derived table flatten count is rendered as a + // trailing breadcrumb when non-zero. + Set rewrites = new LinkedHashSet<>(); + int flattenCount = collectAppliedRewrites(context, rewrites); + if (flattenCount > 0) { + rewrites.add("DERIVED TABLE FLATTENED " + flattenCount); + } + if (!rewrites.isEmpty()) { + builder.setRewrites(new ArrayList<>(rewrites)); + } + } + + /** + * Populate the plan-total estimate attributes on the supplied builder from the given plan. Should + * be invoked only for a root plan. + * @param builder the attributes builder to populate (no-op when null) + * @param plan the plan to read estimates from (no-op when null) + */ + public static void populateTopOfPlanEstimates(ExplainPlanAttributesBuilder builder, + StatementPlan plan) throws SQLException { + if (builder == null || plan == null) { + return; + } + builder.setEstimatedRows(plan.getEstimatedRowsToScan()); + builder.setEstimatedSizeInBytes(plan.getEstimatedBytesToScan()); + builder.setEstimateInfoTs(plan.getEstimateInfoTimestamp()); + } + + private static int collectAppliedRewrites(StatementContext context, Set out) { + int flattenCount = context.getDerivedTableFlattenCount(); + out.addAll(context.getAppliedRewrites()); + for (StatementContext sub : context.getSubStatementContexts()) { + flattenCount += collectAppliedRewrites(sub, out); + } + return flattenCount; + } + + /** + * Render the top-of-plan disclosure lines from the populated attributes and insert them at the + * head of {@code planSteps}. + * @param planSteps the rendered plan step lines to prepend onto (no-op when null) + * @param attrs the already-populated root plan attributes (no-op when null) + */ + public static void renderTopOfPlanText(List planSteps, ExplainPlanAttributes attrs) { + if (planSteps == null || attrs == null) { + return; + } + List lines = new ArrayList<>(); + if (attrs.getTenantId() != null) { + lines.add("TENANT '" + attrs.getTenantId() + "'"); + } + if (attrs.getViewName() != null) { + StringBuilder viewLine = new StringBuilder("VIEW ").append(attrs.getViewName()); + if (attrs.getViewBaseName() != null) { + viewLine.append(" OVER ").append(attrs.getViewBaseName()); + } + lines.add(viewLine.toString()); + } + if (attrs.getCdcScopes() != null) { + lines.add("CDC SCOPE " + attrs.getCdcScopes()); + } + if (attrs.getTxnProvider() != null) { + lines.add("TXN " + attrs.getTxnProvider()); + } + if (attrs.getRewrites() != null) { + for (String rewrite : attrs.getRewrites()) { + lines.add("REWRITE " + rewrite); + } + } + planSteps.addAll(0, lines); + } + protected void explain(String prefix, List planSteps, ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List regionLocations) { StringBuilder buf = new StringBuilder(prefix); ScanRanges scanRanges = context.getScanRanges(); Scan scan = context.getScan(); + boolean verbose = context.isVerbose(); if (scan.getConsistency() != Consistency.STRONG) { buf.append("TIMELINE-CONSISTENCY "); @@ -150,15 +328,7 @@ protected void explain(String prefix, List planSteps, String tableName = tableRef.getTable().getPhysicalName().getString(); if (tableRef.getTable().getIndexType() == PTable.IndexType.LOCAL) { - String indexName = tableRef.getTable().getName().getString(); - if ( - tableRef.getTable().getViewIndexId() != null - && indexName.contains(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR) - ) { - int lastIndexOf = indexName.lastIndexOf(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR); - indexName = indexName.substring(lastIndexOf + 1); - } - tableName = indexName + "(" + tableName + ")"; + tableName = getExplainIndexName(tableRef.getTable()) + "(" + tableName + ")"; } buf.append("OVER ").append(tableName); @@ -174,12 +344,84 @@ protected void explain(String prefix, List planSteps, if (OrderBy.REV_ROW_KEY_ORDER_BY.equals(orderBy)) { explainPlanAttributesBuilder.setClientSortedBy("REVERSE"); } - explainPlanAttributesBuilder.setExplainScanType(scanTypeDetails); + explainPlanAttributesBuilder.setExplainScanType(scanTypeDetails.trim()); explainPlanAttributesBuilder.setTableName(tableName); if (!scanRanges.isPointLookup()) { - explainPlanAttributesBuilder.setKeyRanges(appendKeyRanges()); + explainPlanAttributesBuilder.setKeyRanges(appendKeyRanges().trim()); + } + } + + emitProject(planSteps, explainPlanAttributesBuilder, verbose); + + PTable.IndexType indexType = tableRef.getTable().getIndexType(); + String explainIndexName = getExplainIndexName(tableRef.getTable()); + String indexKind = null; + if (indexType != null) { + switch (indexType) { + case LOCAL: + indexKind = "LOCAL"; + break; + case GLOBAL: + indexKind = "GLOBAL"; + break; + case UNCOVERED_GLOBAL: + indexKind = "UNCOVERED GLOBAL"; + break; + default: + indexKind = null; + } + } + OptimizerDecision decision = getOptimizerDecision(); + StringBuilder indexLine = new StringBuilder(" INDEX ").append(explainIndexName); + if (indexKind != null) { + indexLine.append(" ").append(indexKind); + } + if (decision != null) { + // Disclose the selection rule (unless it is a suppressed default) and, when the chosen plan + // is a functional index that matched a query expression, the separate "matches " + // disclosure. Both may appear together as "/* , matches */". + boolean showRule = !isDefaultRule(decision.getRule()); + String functionalMatch = decision.getFunctionalMatch(); + if (showRule || functionalMatch != null) { + indexLine.append(" /* "); + if (showRule) { + indexLine.append(decision.getRule()); + } + if (functionalMatch != null) { + if (showRule) { + indexLine.append(", "); + } + indexLine.append(functionalMatch); + } + indexLine.append(" */"); + } + } + planSteps.add(indexLine.toString()); + if (verbose && decision != null) { + for (RejectedIndexEntry rejected : decision.getRejectedIndexes()) { + planSteps + .add(" /* !INDEX " + rejected.getName() + " -- " + rejected.getReason() + " */"); + } + } + Integer bucketNum = tableRef.getTable().getBucketNum(); + if (bucketNum != null) { + planSteps.add(" SALT BUCKETS " + bucketNum); + } + int splitCount = getSplitCount(); + if (splitCount > 0) { + planSteps.add(" REGIONS PLANNED " + splitCount); + } + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setIndexName(explainIndexName); + explainPlanAttributesBuilder.setIndexKind(indexKind); + if (bucketNum != null) { + explainPlanAttributesBuilder.setSaltBuckets(bucketNum); + } + if (splitCount > 0) { + explainPlanAttributesBuilder.setRegionsPlanned(splitCount); } } + if (context.getScan() != null && tableRef.getTable().getRowTimestampColPos() != -1) { TimeRange range = context.getScan().getTimeRange(); planSteps.add(" ROW TIMESTAMP FILTER [" + range.getMin() + ", " + range.getMax() + ")"); @@ -231,23 +473,29 @@ protected void explain(String prefix, List planSteps, whereFilterStr = Bytes.toString(expBytes); } } - if (whereFilterStr != null) { - String serverWhereFilter = - "SERVER FILTER BY " + (firstKeyOnlyFilter == null ? "" : "FIRST KEY ONLY AND ") - + (emptyColumnOnlyFilter == null ? "" : "EMPTY COLUMN ONLY AND ") + whereFilterStr; - planSteps.add(" " + serverWhereFilter); + if (firstKeyOnlyFilter != null) { + planSteps.add(" SERVER PROJECTION FILTER BY FIRST KEY ONLY"); if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder.setServerWhereFilter(serverWhereFilter); + explainPlanAttributesBuilder.setServerFirstKeyOnlyProjection(true); } - } else if (firstKeyOnlyFilter != null) { - planSteps.add(" SERVER FILTER BY FIRST KEY ONLY"); + } + if (emptyColumnOnlyFilter != null) { + planSteps.add(" SERVER PROJECTION FILTER BY EMPTY COLUMN ONLY"); if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder.setServerWhereFilter("SERVER FILTER BY FIRST KEY ONLY"); + explainPlanAttributesBuilder.setServerEmptyColumnOnlyProjection(true); } - } else if (emptyColumnOnlyFilter != null) { - planSteps.add(" SERVER FILTER BY EMPTY COLUMN ONLY"); - if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder.setServerWhereFilter("SERVER FILTER BY EMPTY COLUMN ONLY"); + } + emitIgnoredHints(planSteps, explainPlanAttributesBuilder, verbose); + if (whereFilterStr != null) { + if (verbose) { + emitServerFilters(planSteps, explainPlanAttributesBuilder, + whereFilter == null ? null : whereFilter.getExpression(), whereFilterStr); + } else { + String serverWhereFilter = "SERVER FILTER BY " + whereFilterStr; + planSteps.add(" " + serverWhereFilter); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setServerWhereFilter(serverWhereFilter); + } } } if (distinctFilter != null) { @@ -288,8 +536,11 @@ protected void explain(String prefix, List planSteps, } if (explainPlanAttributesBuilder != null) { explainPlanAttributesBuilder.setServerOffset(offset); - if (pageFilter != null) { - explainPlanAttributesBuilder.setServerRowLimit(pageFilter.getPageSize()); + // Populate the attribute whenever a "SERVER n ROW LIMIT" step is emitted, including the + // uncovered-index/server-merge path where the limit originates from the INDEX_LIMIT scan + // attribute. + if (limit != null) { + explainPlanAttributesBuilder.setServerRowLimit(limit); } } } @@ -300,18 +551,286 @@ protected void explain(String prefix, List planSteps, } getRegionLocations(planSteps, explainPlanAttributesBuilder, regionLocations); groupBy.explain(planSteps, groupByLimit, explainPlanAttributesBuilder); - if (scan.getAttribute(BaseScannerRegionObserverConstants.SPECIFIC_ARRAY_INDEX) != null) { - planSteps.add(" SERVER ARRAY ELEMENT PROJECTION"); + emitServerProjection(planSteps, explainPlanAttributesBuilder, "ARRAY", + Collections.singletonList(BaseScannerRegionObserverConstants.SPECIFIC_ARRAY_INDEX)); + emitServerProjection(planSteps, explainPlanAttributesBuilder, "JSON", + Arrays.asList(BaseScannerRegionObserverConstants.JSON_VALUE_FUNCTION, + BaseScannerRegionObserverConstants.JSON_QUERY_FUNCTION)); + emitServerProjection(planSteps, explainPlanAttributesBuilder, "BSON", + Collections.singletonList(BaseScannerRegionObserverConstants.BSON_VALUE_FUNCTION)); + } + + /** + * Emit a {@code SERVER } rewrite breadcrumb on the chosen plan's + * context for every functional-index substitution that actually matched a query expression. The + * pairs were captured at index-candidate-build time (see {@code addPlan} and + * {@code rewriteQueryWithIndexReplacement}) and are looked up here only for the winner, so + * unmatched expressions and unchosen indexes never produce breadcrumbs. + */ + private static void recordFunctionalIndexExpressionBreadcrumbs(QueryPlan winner) { + String tableName = winner.getTableRef().getTable().getTableName().getString(); + Map pairs = winner.getContext().getAppliedIndexExpressionPairs(tableName); + if (pairs.isEmpty()) { + return; + } + for (Map.Entry entry : pairs.entrySet()) { + // Skip if any caller already recorded an equivalent breadcrumb for this winner. + String breadcrumb = "INDEX EXPRESSION " + entry.getValue() + " AS " + entry.getKey(); + if (!winner.getContext().getAppliedRewrites().contains(breadcrumb)) { + winner.getContext().addAppliedRewrite(breadcrumb); + } + } + } + + /** + * Returns {@link OptimizerReasons#matches(String)} for {@code winner} when it is a functional + * index whose indexed expression actually substituted a path expression in the user's query, or + * {@code null} when the functional index rule override does not apply. The applied matches are + * only ever recorded for genuine expression substitutions, so a non-empty match list implies the + * chosen plan is a functional index that matched. + */ + private static String functionalIndexRule(QueryPlan winner) { + PTable table = winner.getTableRef().getTable(); + List matches = + winner.getContext().getAppliedIndexExpressionMatches(table.getTableName().getString()); + if (matches.isEmpty()) { + return null; + } + return OptimizerReasons.matches(matches.get(0)); + } + + /** + * Swaps a generic coverage driven rejection reason for + * {@link OptimizerReasons#REASON_PATH_EXPRESSION_DOES_NOT_MATCH} when {@code index} is a + * functional index for which no query expression matched its indexed expression. Other reasons, + * and functional indexes that did match an expression, keep their original reason. + */ + private static String adjustReasonForFunctionalIndex(PTable index, String reason, + StatementContext rewriteContext) { + if (!rewriteContext.isFunctionalIndex(index.getTableName().getString())) { + return reason; + } + if ( + !OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION.equals(reason) + && !OptimizerReasons.REASON_NO_PK_PREFIX_BOUND.equals(reason) + && !OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE.equals(reason) + ) { + return reason; + } + if ( + !rewriteContext.getAppliedIndexExpressionMatches(index.getTableName().getString()).isEmpty() + ) { + return reason; + } + return OptimizerReasons.REASON_PATH_EXPRESSION_DOES_NOT_MATCH; + } + + /** First plan in {@code plans} that is not {@code exclude}, or {@code null} if there is none. */ + private static QueryPlan firstOther(List plans, QueryPlan exclude) { + for (QueryPlan plan : plans) { + if (plan != exclude) { + return plan; + } + } + return null; + } + + /** The bound-PK-column count for {@code plan}. */ + private static int adjustedBoundCount(QueryPlan plan, int boundRanges) { + PTable table = plan.getTableRef().getTable(); + int boundCount = plan.getContext().getScanRanges().getBoundPkColumnCount(); + boundCount += table.getViewIndexId() == null ? 0 : (boundRanges - 1); + boundCount -= plan.getContext().getScanRanges().isSalted() ? 1 : 0; + return boundCount; + } + + /** + * Returns the {@code RULE_*} label for the chosen plan by replaying the comparator ladder in + * {@link #orderPlansBestToWorst} against the runner up and returning the first decisive branch. + */ + private static String labelComparatorRule(QueryPlan winner, QueryPlan runnerUp, int boundRanges) { + if (runnerUp == null) { + return OptimizerReasons.RULE_ONLY_CANDIDATE; + } + PTable winnerTable = winner.getTableRef().getTable(); + PTable runnerUpTable = runnerUp.getTableRef().getTable(); + if (adjustedBoundCount(winner, boundRanges) != adjustedBoundCount(runnerUp, boundRanges)) { + return OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS; + } + if (winner.getGroupBy() != null && runnerUp.getGroupBy() != null) { + if (winner.getGroupBy().isOrderPreserving() != runnerUp.getGroupBy().isOrderPreserving()) { + return OptimizerReasons.RULE_ORDER_PRESERVING; + } + } + if (winnerTable.getIndexWhere() != null && runnerUpTable.getIndexWhere() == null) { + return OptimizerReasons.RULE_PARTIAL_INDEX_APPLICABLE; + } + return OptimizerReasons.RULE_NON_LOCAL_PREFERRED; + } + + /** Tags every index candidate that lost to {@code winner} with a rejection reason. */ + private static void tagComparatorRejections(QueryPlan winner, List plans, + int boundRanges, DecisionState state) { + int winnerBound = adjustedBoundCount(winner, boundRanges); + boolean winnerNonLocal = winner.getTableRef().getTable().getIndexType() != IndexType.LOCAL; + for (QueryPlan plan : plans) { + if (plan == winner) { + continue; + } + PTable table = plan.getTableRef().getTable(); + if (table.getType() != PTableType.INDEX) { + continue; + } + if (adjustedBoundCount(plan, boundRanges) < winnerBound) { + state.reject(table, adjustReasonForFunctionalIndex(table, + OptimizerReasons.REASON_NO_PK_PREFIX_BOUND, winner.getContext())); + } else if (table.getIndexType() == IndexType.LOCAL && winnerNonLocal) { + state.reject(table, adjustReasonForFunctionalIndex(table, + OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE, winner.getContext())); + } + } + } + private static class WhereConditionRewriter extends AndRewriterBooleanParseNodeVisitor { private final ColumnResolver dataResolver; private final ExpressionCompiler expressionCompiler; @@ -830,7 +1144,8 @@ protected ParseNode leaveBooleanNode(ParseNode node, List l) throws S private static SelectStatement rewriteQueryWithIndexReplacement( final PhoenixConnection connection, final ColumnResolver resolver, final SelectStatement select, - final Map replacement) throws SQLException { + final Map replacement, final StatementContext breadcrumbContext) + throws SQLException { TableNode from = select.getFrom(); TableNode newFrom = from.accept(new QueryOptimizerTableNode(resolver, replacement)); if (from == newFrom) { @@ -841,9 +1156,23 @@ private static SelectStatement rewriteQueryWithIndexReplacement( IndexStatementRewriter.translate(FACTORY.select(select, newFrom), resolver, replacement); for (TableRef indexTableRef : replacement.values()) { // replace expressions with corresponding matching columns for functional indexes - indexSelect = ParseNodeRewriter.rewrite(indexSelect, + IndexExpressionParseNodeRewriter indexExpressionRewriter = new IndexExpressionParseNodeRewriter(indexTableRef.getTable(), - indexTableRef.getTableAlias(), connection, indexSelect.getUdfParseNodes())); + indexTableRef.getTableAlias(), connection, indexSelect.getUdfParseNodes(), + breadcrumbContext); + indexSelect = ParseNodeRewriter.rewrite(indexSelect, indexExpressionRewriter); + // Surface which functional index expressions actually matched so the optimizer can label + // the chosen index's rule and distinguish functional index rejections that matched nothing. + if (indexExpressionRewriter.hasFunctionalColumns()) { + breadcrumbContext + .recordFunctionalIndex(indexTableRef.getTable().getTableName().getString()); + } + breadcrumbContext.recordAppliedIndexExpressionMatches( + indexTableRef.getTable().getTableName().getString(), + indexExpressionRewriter.getAppliedFunctionalSubstitutions().values()); + breadcrumbContext.recordAppliedIndexExpressionPairs( + indexTableRef.getTable().getTableName().getString(), + indexExpressionRewriter.getAppliedFunctionalSubstitutions()); } return indexSelect; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/RejectedIndexEntry.java b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/RejectedIndexEntry.java new file mode 100644 index 00000000000..81568ac7c93 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/RejectedIndexEntry.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.optimize; + +import java.util.Objects; + +/** + * An index the optimizer considered as a candidate but did not choose, paired with the reason it + * was rejected. The reason is one of the closed-set {@code REASON_*} labels in + * {@link OptimizerReasons}. + */ +public final class RejectedIndexEntry { + private final String name; + private final String reason; + + public RejectedIndexEntry(String name, String reason) { + this.name = Objects.requireNonNull(name, "name"); + this.reason = Objects.requireNonNull(reason, "reason"); + } + + public String getName() { + return name; + } + + public String getReason() { + return reason; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RejectedIndexEntry)) { + return false; + } + RejectedIndexEntry that = (RejectedIndexEntry) o; + return name.equals(that.name) && reason.equals(that.reason); + } + + @Override + public int hashCode() { + return Objects.hash(name, reason); + } + + @Override + public String toString() { + return "RejectedIndexEntry{name=" + name + ", reason=" + reason + "}"; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainOptions.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainOptions.java new file mode 100644 index 00000000000..95b1bce97ce --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainOptions.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.parse; + +import java.util.Objects; + +/** + * POJO carrying the options parsed from an {@code EXPLAIN} statement's option list + * ({@code EXPLAIN [( [, ]*)] }). + */ +public final class ExplainOptions { + + /** Output format selected by the {@code FORMAT} option. */ + public enum Format { + TEXT, + JSON + } + + public static final ExplainOptions DEFAULT = new ExplainOptions(false, false, Format.TEXT); + public static final ExplainOptions WITH_REGIONS = new ExplainOptions(true, false, Format.TEXT); + public static final ExplainOptions VERBOSE = new ExplainOptions(false, true, Format.TEXT); + + private final boolean regions; + private final boolean verbose; + private final Format format; + + public ExplainOptions(boolean regions, boolean verbose, Format format) { + this.regions = regions; + this.verbose = verbose; + this.format = format == null ? Format.TEXT : format; + } + + public boolean isRegions() { + return regions; + } + + public boolean isVerbose() { + return verbose; + } + + public Format getFormat() { + return format; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ExplainOptions)) { + return false; + } + ExplainOptions that = (ExplainOptions) o; + return regions == that.regions && verbose == that.verbose && format == that.format; + } + + @Override + public int hashCode() { + return Objects.hash(regions, verbose, format); + } + + @Override + public String toString() { + return "ExplainOptions{regions=" + regions + ", verbose=" + verbose + ", format=" + format + + "}"; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainStatement.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainStatement.java index a92696f67bf..981cff6ef6c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainStatement.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainStatement.java @@ -21,11 +21,11 @@ public class ExplainStatement implements BindableStatement { private final BindableStatement statement; - private final ExplainType explainType; + private final ExplainOptions options; - public ExplainStatement(BindableStatement statement, ExplainType explainType) { + public ExplainStatement(BindableStatement statement, ExplainOptions options) { this.statement = statement; - this.explainType = explainType; + this.options = options == null ? ExplainOptions.DEFAULT : options; } public BindableStatement getStatement() { @@ -42,7 +42,7 @@ public Operation getOperation() { return Operation.QUERY; } - public ExplainType getExplainType() { - return explainType; + public ExplainOptions getOptions() { + return options; } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainType.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainType.java deleted file mode 100644 index 883e3c6a90c..00000000000 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainType.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.parse; - -/** - * Explain type attributes used to differentiate output of the explain plan. - */ -public enum ExplainType { - WITH_REGIONS, - DEFAULT -} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java index 67085741d54..8dab318ffbe 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java @@ -19,6 +19,7 @@ import java.sql.SQLException; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.apache.phoenix.compile.ColumnResolver; @@ -44,10 +45,37 @@ public class IndexExpressionParseNodeRewriter extends ParseNodeRewriter { private final Map indexedParseNodeToColumnParseNodeMap; + /** + * Maps the rewritten parse node of a functional index column to a [colName, expressionStr] pair. + * Plain PK columns are excluded. + */ + private final Map indexedParseNodeToFunctionalColumn; + + /** + * Records the functional index substitutions that actually fired against the query, mapping the + * index column name to the expression string. Insertion order is preserved so callers can report + * matches in first-seen order. + */ + private final Map appliedFunctionalSubstitutions = new LinkedHashMap<>(); + public IndexExpressionParseNodeRewriter(PTable index, String alias, PhoenixConnection connection, Map udfParseNodes) throws SQLException { + this(index, alias, connection, udfParseNodes, null); + } + + /** + * Retained for API compatibility. The {@code breadcrumbContext} parameter is no longer used: + * {@code REWRITE INDEX EXPRESSION ... AS ...} breadcrumbs are now emitted by the optimizer after + * a winner is selected, against {@link #getAppliedFunctionalSubstitutions()}, so the breadcrumb + * fires once per substitution that actually matched a query expression and only for the chosen + * index plan rather than for every PK column of every candidate index. + */ + public IndexExpressionParseNodeRewriter(PTable index, String alias, PhoenixConnection connection, + Map udfParseNodes, StatementContext breadcrumbContext) + throws SQLException { indexedParseNodeToColumnParseNodeMap = Maps.newHashMapWithExpectedSize(index.getColumns().size()); + indexedParseNodeToFunctionalColumn = Maps.newHashMapWithExpectedSize(index.getColumns().size()); NamedTableNode tableNode = NamedTableNode.create(alias, TableName.create(index.getParentSchemaName().getString(), index.getParentTableName().getString()), Collections. emptyList()); @@ -74,15 +102,46 @@ public IndexExpressionParseNodeRewriter(PTable index, String alias, PhoenixConne columnParseNode = NODE_FACTORY.cast(columnParseNode, expressionDataType, null, null); } indexedParseNodeToColumnParseNodeMap.put(indexedParseNode, columnParseNode); + // Only true functional columns defined over an expression get an applied match entry. A + // plain indexed or PK column's expression string parses to a bare column reference. An + // expression column (e.g. UPPER(NAME)) parses to a compound node. + if (!(expressionParseNode instanceof ColumnParseNode)) { + // Trim leading/trailing whitespace + indexedParseNodeToFunctionalColumn.put(indexedParseNode, + new String[] { colName, expressionStr.trim() }); + } } } @Override protected ParseNode leaveCompoundNode(CompoundParseNode node, List children, CompoundNodeFactory factory) { - return indexedParseNodeToColumnParseNodeMap.containsKey(node) - ? indexedParseNodeToColumnParseNodeMap.get(node) - : super.leaveCompoundNode(node, children, factory); + ParseNode replacement = indexedParseNodeToColumnParseNodeMap.get(node); + if (replacement == null) { + return super.leaveCompoundNode(node, children, factory); + } + // A functional index substitution actually fired for this query node. + String[] functionalColumn = indexedParseNodeToFunctionalColumn.get(node); + if (functionalColumn != null) { + appliedFunctionalSubstitutions.put(functionalColumn[0], functionalColumn[1]); + } + return replacement; + } + + /** + * Returns the functional index substitutions that actually fired against the query, keyed by + * index column name with the expression string as the value, in first-seen order. + */ + public Map getAppliedFunctionalSubstitutions() { + return Collections.unmodifiableMap(appliedFunctionalSubstitutions); + } + + /** + * Returns {@code true} if the index this rewriter was built for has at least one functional + * column, regardless of whether any such column matched the query. + */ + public boolean hasFunctionalColumns() { + return !indexedParseNodeToFunctionalColumn.isEmpty(); } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java index a5070c79238..2a36dc40355 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java @@ -219,8 +219,8 @@ public static String createTempAlias() { return "$" + tempAliasCounter.incrementAndGet(); } - public ExplainStatement explain(BindableStatement statement, ExplainType explainType) { - return new ExplainStatement(statement, explainType); + public ExplainStatement explain(BindableStatement statement, ExplainOptions options) { + return new ExplainStatement(statement, options); } public AliasedNode aliasedNode(String alias, ParseNode expression) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 22833e24945..1d7ff320f0d 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -1755,8 +1755,15 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits // can lose information during compilation StringBuilder buf = new StringBuilder(); parseNode.toSQL(resolver, buf); + // Several ParseNode.toSQL implementations (FunctionParseNode, CastParseNode, etc.) + // unconditionally prepend a separator space so the serialized form composes safely + // when nested inside a larger expression. At the top of an index expression there is + // no surrounding context, so trim the resulting string before it becomes part of the + // functional index column name. Otherwise the leading space would be baked into the + // PColumn name and would later show up inside the double quotes of the case-sensitive + // display name (e.g. PROJECT " JSON_VALUE(DOC,'$.type')"). // need to escape backslash as this expression will be re-parsed later - String expressionStr = StringUtil.escapeBackslash(buf.toString()); + String expressionStr = StringUtil.escapeBackslash(buf.toString().trim()); ColumnName colName = null; ColumnRef colRef = expressionIndexCompiler.getColumnRef(); @@ -4923,9 +4930,9 @@ public MutationState addColumn(PTable table, List origColumnDefs, /** * To check if TTL is defined at any of the child below we are checking it at * {@link org.apache.phoenix.coprocessor.MetaDataEndpointImpl#mutateColumn(List, ColumnMutator, int, PTable, PTable, boolean)} - * level where in function - * {@link org.apache.phoenix.coprocessor.MetaDataEndpointImpl# validateIfMutationAllowedOnParent(PTable, List, PTableType, long, byte[], byte[], byte[], List, int)} - * we are already traversing through allDescendantViews. + * level where in function {@link org.apache.phoenix.coprocessor.MetaDataEndpointImpl# + * validateIfMutationAllowedOnParent(PTable, List, PTableType, long, byte[], byte[], + * byte[], List, int)} we are already traversing through allDescendantViews. */ } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java index dee96ceafc8..89f6fc7ad76 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -2537,6 +2537,8 @@ private void buildIndexWhereExpression(PhoenixConnection connection) throws SQLE ParseNode where = plan.getStatement().getWhere(); plan.getContext().setResolver(FromCompiler.getResolver(plan.getTableRef())); indexWhereExpression = transformDNF(where, plan.getContext()); + // Tag the partial-index WHERE predicate with its origin for VERBOSE attribution. + plan.getContext().tagPredicate(indexWhereExpression, "INDEX WHERE"); indexWhereColumns = Sets.newHashSetWithExpectedSize(plan.getContext().getWhereConditionColumns().size()); for (Pair column : plan.getContext().getWhereConditionColumns()) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ParseNodeUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ParseNodeUtil.java index 4d96f1bf9cc..e3c1b83aa65 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ParseNodeUtil.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ParseNodeUtil.java @@ -23,6 +23,7 @@ import org.apache.phoenix.compile.ColumnResolver; import org.apache.phoenix.compile.FromCompiler; import org.apache.phoenix.compile.QueryCompiler; +import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.compile.StatementNormalizer; import org.apache.phoenix.compile.SubqueryRewriter; import org.apache.phoenix.compile.SubselectRewriter; @@ -159,16 +160,20 @@ public ColumnResolver getColumnResolver() { /** * Optimize rewriting {@link SelectStatement} by {@link SubselectRewriter} and * {@link SubqueryRewriter} before {@link QueryCompiler#compile}. + *

+ * The supplied {@link StatementContext} is the top level context that carries top-of-plan rewrite + * breadcrumb. */ - public static RewriteResult rewrite(SelectStatement selectStatement, - PhoenixConnection phoenixConnection) throws SQLException { + public static RewriteResult rewrite(SelectStatement selectStatement, StatementContext context) + throws SQLException { + PhoenixConnection phoenixConnection = context.getConnection(); SelectStatement selectStatementToUse = - SubselectRewriter.flatten(selectStatement, phoenixConnection); + SubselectRewriter.flatten(selectStatement, phoenixConnection, context); ColumnResolver columnResolver = FromCompiler.getResolverForQuery(selectStatementToUse, phoenixConnection); selectStatementToUse = StatementNormalizer.normalize(selectStatementToUse, columnResolver); SelectStatement transformedSubquery = - SubqueryRewriter.transform(selectStatementToUse, columnResolver, phoenixConnection); + SubqueryRewriter.transform(selectStatementToUse, columnResolver, phoenixConnection, context); if (transformedSubquery != selectStatementToUse) { columnResolver = FromCompiler.getResolverForQuery(transformedSubquery, phoenixConnection); transformedSubquery = StatementNormalizer.normalize(transformedSubquery, columnResolver); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterSessionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterSessionIT.java index 1fe2d15e799..0f02a5b7a36 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterSessionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterSessionIT.java @@ -17,20 +17,18 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.Statement; import java.util.Properties; import org.apache.hadoop.hbase.client.Consistency; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -58,16 +56,12 @@ public void testUpdateConsistency() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl(), props)) { Statement st = conn.createStatement(); st.execute("alter session set Consistency = 'timeline'"); - ResultSet rs = st.executeQuery("explain select * from " + tableName); assertEquals(Consistency.TIMELINE, conn.unwrap(PhoenixConnection.class).getConsistency()); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(queryPlan.indexOf("TIMELINE") > 0); + assertPlan(conn, "select * from " + tableName).consistency("TIMELINE"); // turn off timeline read consistency st.execute("alter session set Consistency = 'strong'"); - rs = st.executeQuery("explain select * from " + tableName); - queryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(queryPlan.indexOf("TIMELINE") < 0); + assertPlan(conn, "select * from " + tableName).consistency("STRONG"); } } @@ -77,10 +71,7 @@ public void testSetConsistencyInURL() throws Exception { try (Connection conn = DriverManager.getConnection( getUrl() + PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR + "Consistency=TIMELINE", props)) { assertEquals(Consistency.TIMELINE, ((PhoenixConnection) conn).getConsistency()); - Statement st = conn.createStatement(); - ResultSet rs = st.executeQuery("explain select * from " + tableName); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(queryPlan.indexOf("TIMELINE") > 0); + assertPlan(conn, "select * from " + tableName).consistency("TIMELINE"); conn.close(); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java index fde20a61309..8c5705b7876 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlanWithRegions; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.assertResultSet; import static org.junit.Assert.assertEquals; @@ -39,10 +41,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; @@ -513,20 +512,14 @@ public void testGroupByOrderPreserving() throws Exception { assertEquals("abc", rs.getString(2)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals( - " ['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]", - explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals( - "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]", - explainPlanAttributes.getServerAggregate()); - assertEquals("COUNT(1) > 1", explainPlanAttributes.getClientFilterBy()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(tableName) + .keyRanges( + "['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]") + .serverFirstKeyOnlyProjection(true) + .serverAggregate( + "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]") + .clientFilterBy("COUNT(1) > 1"); } @Test @@ -564,18 +557,11 @@ public void testGroupByOrderPreservingDescSort() throws Exception { assertEquals("a", rs.getString(1)); assertEquals(4, rs.getLong(2)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", - explainPlanAttributes.getServerAggregate()); - assertFalse("Explain plan regionLocation attribute should not be empty", - explainPlanAttributes.getRegionLocations().isEmpty()); + assertPlanWithRegions(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY") + .clientSortedBy("REVERSE").scanType("FULL SCAN").table(tableName) + .serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]") + .regionLocationsNotEmpty(); } @Test @@ -621,18 +607,11 @@ public void testSumGroupByOrderPreservingDesc() throws Exception { assertEquals("a", rs.getString(1)); assertEquals(10, rs.getLong(2)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", - explainPlanAttributes.getServerAggregate()); - assertFalse("Explain plan regionLocation attribute should not be empty", - explainPlanAttributes.getRegionLocations().isEmpty()); + assertPlanWithRegions(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY") + .clientSortedBy("REVERSE").scanType("FULL SCAN").table(tableName) + .serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]") + .regionLocationsNotEmpty(); } @Test @@ -688,21 +667,14 @@ protected void testAvgGroupByOrderPreserving(Connection conn, String tableName, assertEquals("n", rs.getString(1)); assertEquals(2, rs.getDouble(2), 1e-6); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", - explainPlanAttributes.getServerAggregate()); + assertPlanWithRegions(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY") + .scanType("FULL SCAN").table(tableName).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]") + .regionLocationsNotEmpty(); TestUtil.analyzeTable(conn, tableName); List splits = TestUtil.getAllSplits(conn, tableName); // nGuideposts when stats are enabled, 4 when disabled assertEquals(4, splits.size()); - assertFalse("Explain plan regionLocation attribute should not be empty", - explainPlanAttributes.getRegionLocations().isEmpty()); } @Test @@ -833,9 +805,6 @@ public void testGroupByOrderByDescBug3451() throws Exception { assertEquals("entityId3", rs.getString(1)); assertEquals(1.4, rs.getDouble(2), 0.001); assertFalse(rs.next()); - - String expectedPhoenixPlan = ""; - validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves2IT.java index 6ae7869cfe5..486ae3d3825 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves2IT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -31,9 +32,6 @@ import java.sql.SQLException; import java.util.List; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.types.PChar; @@ -137,20 +135,14 @@ public void testGroupByOrderPreserving() throws Exception { assertEquals("abc", rs.getString(2)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals( - " ['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]", - explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals( - "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]", - explainPlanAttributes.getServerAggregate()); - assertEquals("COUNT(1) > 1", explainPlanAttributes.getClientFilterBy()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(tableName) + .keyRanges( + "['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]") + .serverFirstKeyOnlyProjection(true) + .serverAggregate( + "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]") + .clientFilterBy("COUNT(1) > 1"); } @Test @@ -190,16 +182,9 @@ public void testGroupByOrderPreservingDescSort() throws Exception { assertEquals("a", rs.getString(1)); assertEquals(4, rs.getLong(2)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", - explainPlanAttributes.getServerAggregate()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").clientSortedBy("REVERSE") + .scanType("FULL SCAN").table(tableName).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]"); } @Test @@ -249,16 +234,9 @@ public void testSumGroupByOrderPreservingDesc() throws Exception { assertEquals("a", rs.getString(1)); assertEquals(10, rs.getLong(2)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", - explainPlanAttributes.getServerAggregate()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").clientSortedBy("REVERSE") + .scanType("FULL SCAN").table(tableName).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]"); } @Test @@ -304,13 +282,8 @@ public void testSumGroupByOrderPreservingDescWithoutSplit() throws Exception { assertEquals("a", rs.getString(1)); assertEquals(10, rs.getLong(2)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").clientSortedBy("REVERSE") + .scanType("RANGE SCAN").table(tableName); } @Test @@ -463,15 +436,9 @@ protected void testAvgGroupByOrderPreserving(Connection conn, String tableName, assertEquals("n", rs.getString(1)); assertEquals(2, rs.getDouble(2), 1e-6); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", - explainPlanAttributes.getServerAggregate()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(tableName).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]"); TestUtil.analyzeTable(conn, tableName); List splits = TestUtil.getAllSplits(conn, tableName); // nGuideposts when stats are enabled, 4 when disabled @@ -527,9 +494,6 @@ public void testGroupByOrderByDescBug3451() throws Exception { assertEquals("entityId3", rs.getString(1)); assertEquals(1.4, rs.getDouble(2), 0.001); assertFalse(rs.next()); - - String expectedPhoenixPlan = ""; - validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMovesIT.java index 621a3d33805..83910c50ed9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMovesIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -32,9 +33,6 @@ import java.sql.Statement; import java.util.List; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.types.PChar; @@ -365,15 +363,9 @@ protected void testAvgGroupByOrderPreserving(Connection conn, String tableName, assertEquals("n", rs.getString(1)); assertEquals(2, rs.getDouble(2), 1e-6); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", - explainPlanAttributes.getServerAggregate()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(tableName).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]"); TestUtil.analyzeTable(conn, tableName); List splits = TestUtil.getAllSplits(conn, tableName); // nGuideposts when stats are enabled, 4 when disabled @@ -402,15 +394,9 @@ protected void testAvgGroupByDescOrderPreserving(Connection conn, String tableNa assertEquals("a", rs.getString(1)); assertEquals(3, rs.getDouble(2), 1e-6); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", - explainPlanAttributes.getServerAggregate()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(tableName).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]"); TestUtil.analyzeTable(conn, tableName); List splits = TestUtil.getAllSplits(conn, tableName); // nGuideposts when stats are enabled, 4 when disabled diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java index 62e7a9271e1..d02a8a7058e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.ROW1; import static org.apache.phoenix.util.TestUtil.ROW2; import static org.apache.phoenix.util.TestUtil.ROW3; @@ -30,7 +31,6 @@ import static org.apache.phoenix.util.TestUtil.assertResultSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.sql.Connection; @@ -40,9 +40,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryBuilder; @@ -290,18 +287,10 @@ public void testAggregateOptimizedOutOrderBy() throws Exception { .setSelectExpression("DISTINCT(K2)").setWhereClause("K2 = 'ABC'"); // verify that the phoenix query plan doesn't contain an order by - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY K2 = 'ABC'", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO DISTINCT ROWS BY [K2, VAL1, VAL2]", - explainPlanAttributes.getServerAggregate()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertNull(explainPlanAttributes.getClientSortedBy()); - assertNull(explainPlanAttributes.getServerSortedBy()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(tableName).serverWhereFilter("SERVER FILTER BY K2 = 'ABC'") + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [K2, VAL1, VAL2]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy(null).serverSortedBy(null); ResultSet rs = executeQuery(conn, queryBuilder); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByWithRegionMovesIT.java index bef4c8aaceb..2fb0d59022b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByWithRegionMovesIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.ROW1; import static org.apache.phoenix.util.TestUtil.ROW2; import static org.apache.phoenix.util.TestUtil.ROW3; @@ -30,7 +31,6 @@ import static org.apache.phoenix.util.TestUtil.assertResultSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.sql.Connection; @@ -40,9 +40,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryBuilder; import org.apache.phoenix.util.SchemaUtil; @@ -362,18 +359,10 @@ public void testAggregateOptimizedOutOrderBy() throws Exception { .setSelectExpression("DISTINCT(K2)").setWhereClause("K2 = 'ABC'"); // verify that the phoenix query plan doesn't contain an order by - ExplainPlan plan = conn.prepareStatement(queryBuilder.build()) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY K2 = 'ABC'", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO DISTINCT ROWS BY [K2, VAL1, VAL2]", - explainPlanAttributes.getServerAggregate()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertNull(explainPlanAttributes.getClientSortedBy()); - assertNull(explainPlanAttributes.getServerSortedBy()); + assertPlan(conn, queryBuilder.build()).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(tableName).serverWhereFilter("SERVER FILTER BY K2 = 'ABC'") + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [K2, VAL1, VAL2]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy(null).serverSortedBy(null); ResultSet rs = executeQuery(conn, queryBuilder); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java index 4f8622aacc9..e3fc2ac9ee0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -85,7 +86,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.phoenix.thirdparty.com.google.common.base.Joiner; import org.apache.phoenix.thirdparty.com.google.common.base.Throwables; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; @@ -754,12 +754,13 @@ public Object run() throws Exception { ResultSet rs = stmt.executeQuery(readTableSQL); assertNotNull(rs); int i = 0; - String explainPlan = Joiner.on(" ") - .join(((PhoenixStatement) stmt).getQueryPlan().getExplainPlan().getPlanSteps()); + String scannedTable = ((PhoenixStatement) stmt).getQueryPlan().getExplainPlan() + .getPlanStepsAsAttributes().getTableName(); rs = stmt.executeQuery(readTableSQL); if (tenantId != null) { rs.next(); - assertFalse(explainPlan.contains("_IDX_")); + assertFalse("expected scanned table <" + scannedTable + "> to not be a view index", + scannedTable != null && scannedTable.contains("_IDX_")); assertEquals(((PhoenixConnection) conn).getTenantId().toString(), tenantId); // For tenant ID "o3", the value in table will be 3 assertEquals(Character.toString(tenantId.charAt(1)), rs.getString(1)); @@ -796,9 +797,7 @@ public Object run() throws Exception { ResultSet rs = stmt.executeQuery(readTableSQL); assertNotNull(rs); int i = 0; - String explainPlan = Joiner.on(" ") - .join(((PhoenixStatement) stmt).getQueryPlan().getExplainPlan().getPlanSteps()); - assertTrue(explainPlan.contains("_IDX_")); + assertPlan(((PhoenixStatement) stmt).getQueryPlan()).tableContains("_IDX_"); rs = stmt.executeQuery(readTableSQL); if (tenantId != null) { rs.next(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java index 539d1316edd..3245419ecbb 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java @@ -17,8 +17,8 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.thirdparty.com.google.common.collect.Sets.newHashSet; -import static org.junit.Assert.assertEquals; import java.sql.Connection; import java.sql.DriverManager; @@ -29,9 +29,7 @@ import java.util.Properties; import java.util.Set; import org.apache.hadoop.hbase.util.Pair; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; @@ -159,22 +157,10 @@ private void createAndVerifyIndex(Connection conn, String viewName, String table } else { conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + viewName + "(v2)"); } + // sanity check that we can upsert after index is there conn.createStatement() - .execute("UPSERT INTO " + viewName + "(k2,v1,v2) VALUES (-1, 'blah', 'superblah')"); // sanity - // check - // that - // we can - // upsert - // after - // index - // is - // there + .execute("UPSERT INTO " + viewName + "(k2,v1,v2) VALUES (-1, 'blah', 'superblah')"); conn.commit(); - ExplainPlan plan = conn - .prepareStatement( - "SELECT k1, k2, v2 FROM " + viewName + " WHERE v2='" + valuePrefix + "v2-1'") - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); final String iteratorTypeAndScanSize; final String clientSortAlgo; final String expectedTableName; @@ -191,17 +177,17 @@ private void createAndVerifyIndex(Connection conn, String viewName, String table SchemaUtil.getTableName(SchemaUtil.getSchemaNameFromFullName(viewName), indexName) + "(" + tableName + ")"; keyRanges = - " [" + (1L + expectedIndexIdOffset) + ",'" + tenantId + "','" + valuePrefix + "v2-1']"; + "[" + (1L + expectedIndexIdOffset) + ",'" + tenantId + "','" + valuePrefix + "v2-1']"; } else { if (saltBuckets == null) { iteratorTypeAndScanSize = "PARALLEL 1-WAY"; clientSortAlgo = null; - keyRanges = " [" + (Short.MIN_VALUE + expectedIndexIdOffset) + ",'" + tenantId + "','" + keyRanges = "[" + (Short.MIN_VALUE + expectedIndexIdOffset) + ",'" + tenantId + "','" + valuePrefix + "v2-1']"; } else { iteratorTypeAndScanSize = "PARALLEL " + saltBuckets + "-WAY"; clientSortAlgo = "CLIENT MERGE SORT"; - keyRanges = " [X'00'," + (Short.MIN_VALUE + expectedIndexIdOffset) + ",'" + tenantId + "','" + keyRanges = "[X'00'," + (Short.MIN_VALUE + expectedIndexIdOffset) + ",'" + tenantId + "','" + valuePrefix + "v2-1'] - [" + PVarbinary.INSTANCE.toStringLiteral(new byte[] { (byte) (saltBuckets - 1) }) + "," + (Short.MIN_VALUE + expectedIndexIdOffset) + ",'" + tenantId + "','" + valuePrefix @@ -209,42 +195,28 @@ private void createAndVerifyIndex(Connection conn, String viewName, String table } expectedTableName = "_IDX_" + tableName; } - assertEquals(iteratorTypeAndScanSize, explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(clientSortAlgo, explainPlanAttributes.getClientSortAlgo()); - assertEquals(expectedTableName, explainPlanAttributes.getTableName()); - assertEquals(keyRanges, explainPlanAttributes.getKeyRanges()); + assertPlan(conn, "SELECT k1, k2, v2 FROM " + viewName + " WHERE v2='" + valuePrefix + "v2-1'") + .iteratorType(iteratorTypeAndScanSize).serverFirstKeyOnlyProjection(true) + .scanType("RANGE SCAN").clientSortAlgo(clientSortAlgo).table(expectedTableName) + .keyRanges(keyRanges).indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); } private void createAndVerifyIndexNonStringTenantId(Connection conn, String viewName, String tableName, String tenantId, String valuePrefix) throws SQLException { String indexName = generateUniqueName(); conn.createStatement().execute("CREATE LOCAL INDEX " + indexName + " ON " + viewName + "(v2)"); + // sanity check that we can upsert after index is there conn.createStatement() - .execute("UPSERT INTO " + viewName + "(k2,v1,v2) VALUES (-1, 'blah', 'superblah')"); // sanity - // check - // that - // we can - // upsert - // after - // index - // is - // there + .execute("UPSERT INTO " + viewName + "(k2,v1,v2) VALUES (-1, 'blah', 'superblah')"); conn.commit(); - ExplainPlan plan = conn - .prepareStatement( - "SELECT k1, k2, v2 FROM " + viewName + " WHERE v2='" + valuePrefix + "v2-1'") - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(SchemaUtil.getTableName(SchemaUtil.getSchemaNameFromFullName(viewName), indexName) - + "(" + tableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [1," + tenantId + ",'" + valuePrefix + "v2-1']", - explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, "SELECT k1, k2, v2 FROM " + viewName + " WHERE v2='" + valuePrefix + "v2-1'") + .iteratorType("PARALLEL 1-WAY").serverFirstKeyOnlyProjection(true).scanType("RANGE SCAN") + .table(SchemaUtil.getTableName(SchemaUtil.getSchemaNameFromFullName(viewName), indexName) + + "(" + tableName + ")") + .keyRanges("[1," + tenantId + ",'" + valuePrefix + "v2-1']") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); } private Connection createTenantConnection(String tenantId) throws SQLException { @@ -253,7 +225,6 @@ private Connection createTenantConnection(String tenantId) throws SQLException { return DriverManager.getConnection(getUrl(), props); } - @SuppressWarnings("unchecked") private void verifyViewData(Connection conn, String viewName, String valuePrefix) throws SQLException { String query = "SELECT k1, k2, v2 FROM " + viewName + " WHERE v2='" + valuePrefix + "v2-1'"; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java index 2ca630f1132..d389ce3e229 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.analyzeTable; import static org.apache.phoenix.util.TestUtil.getAllSplits; import static org.junit.Assert.assertEquals; @@ -38,10 +39,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.util.MetaDataUtil; @@ -200,38 +198,32 @@ protected Pair testUpdatableViewIndex(Integer saltBuckets, boolean assertEquals("bar", rs.getString(4)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); String iteratorTypeAndScanSize; String clientSortAlgo; String expectedTableName; String keyRanges; - String serverFilterBy; + boolean firstKeyOnlyProjection; if (localIndex) { iteratorTypeAndScanSize = "PARALLEL " + (saltBuckets == null ? 1 : saltBuckets) + "-WAY"; expectedTableName = fullTableName; keyRanges = "[1,51]"; clientSortAlgo = "CLIENT MERGE SORT"; - serverFilterBy = "SERVER FILTER BY FIRST KEY ONLY"; + firstKeyOnlyProjection = true; } else { iteratorTypeAndScanSize = saltBuckets == null ? "PARALLEL 1-WAY" : "PARALLEL " + saltBuckets + "-WAY"; expectedTableName = viewIndexPhysicalName; keyRanges = saltBuckets == null - ? " [" + Short.MIN_VALUE + ",51]" - : " [0," + Short.MIN_VALUE + ",51] - [" + (saltBuckets - 1) + "," + Short.MIN_VALUE - + ",51]"; + ? "[" + Short.MIN_VALUE + ",51]" + : "[0," + Short.MIN_VALUE + ",51] - [" + (saltBuckets - 1) + "," + Short.MIN_VALUE + ",51]"; clientSortAlgo = saltBuckets == null ? null : "CLIENT MERGE SORT"; - serverFilterBy = null; + firstKeyOnlyProjection = false; } - assertEquals(iteratorTypeAndScanSize, explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(expectedTableName, explainPlanAttributes.getTableName()); - assertEquals(clientSortAlgo, explainPlanAttributes.getClientSortAlgo()); - assertEquals(keyRanges, explainPlanAttributes.getKeyRanges()); - assertEquals(serverFilterBy, explainPlanAttributes.getServerWhereFilter()); + assertPlan(conn, query).iteratorType(iteratorTypeAndScanSize).scanType("RANGE SCAN") + .table(expectedTableName).clientSortAlgo(clientSortAlgo).keyRanges(keyRanges) + .serverFirstKeyOnlyProjection(firstKeyOnlyProjection).serverEmptyColumnOnlyProjection(false) + .serverWhereFilter(null); String viewIndexName2 = "I_" + generateUniqueName(); if (localIndex) { @@ -260,32 +252,25 @@ protected Pair testUpdatableViewIndex(Integer saltBuckets, boolean assertEquals("foo", rs.getString(3)); assertFalse(rs.next()); - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - String physicalTableName; if (localIndex) { physicalTableName = fullTableName; iteratorTypeAndScanSize = "PARALLEL " + (saltBuckets == null ? 1 : saltBuckets) + "-WAY"; - keyRanges = " [" + (2) + ",'foo']"; + keyRanges = "[" + (2) + ",'foo']"; clientSortAlgo = "CLIENT MERGE SORT"; } else { physicalTableName = viewIndexPhysicalName; iteratorTypeAndScanSize = saltBuckets == null ? "PARALLEL 1-WAY" : "PARALLEL " + saltBuckets + "-WAY"; keyRanges = saltBuckets == null - ? " [" + (Short.MIN_VALUE + 1) + ",'foo']" - : " [0," + (Short.MIN_VALUE + 1) + ",'foo'] - [" + (saltBuckets - 1) + "," + ? "[" + (Short.MIN_VALUE + 1) + ",'foo']" + : "[0," + (Short.MIN_VALUE + 1) + ",'foo'] - [" + (saltBuckets - 1) + "," + (Short.MIN_VALUE + 1) + ",'foo']"; clientSortAlgo = saltBuckets == null ? null : "CLIENT MERGE SORT"; } - assertEquals(physicalTableName, explainPlanAttributes.getTableName()); - assertEquals(iteratorTypeAndScanSize, explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(keyRanges, explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals(clientSortAlgo, explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, query).table(physicalTableName).iteratorType(iteratorTypeAndScanSize) + .scanType("RANGE SCAN").keyRanges(keyRanges).serverFirstKeyOnlyProjection(true) + .clientSortAlgo(clientSortAlgo); conn.close(); return new Pair<>(physicalTableName, scan); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson4IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson4IT.java index 315932bdd09..7c2b93f4d37 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson4IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson4IT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -43,7 +44,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.hbase.index.metrics.GlobalIndexCheckerSource; import org.apache.phoenix.hbase.index.metrics.GlobalIndexCheckerSourceImpl; @@ -1188,21 +1188,15 @@ private static void assertReturnedOldRowResult(PreparedStatement stmt, String js private static void validateExplainPlan(PreparedStatement ps, String tableName, String scanType) throws SQLException { - ExplainPlan plan = ps.unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - validatePlan(tableName, scanType, plan); + assertPlan(ps.unwrap(PhoenixPreparedStatement.class)).table(tableName) + .iteratorType("PARALLEL 1-WAY").scanType(scanType); } private static void validateExplainPlan(Statement stmt, String query, String tableName, String scanType) throws SQLException { ExplainPlan plan = stmt.unwrap(PhoenixStatement.class).optimizeQuery(query).getExplainPlan(); - validatePlan(tableName, scanType, plan); - } - - private static void validatePlan(String tableName, String scanType, ExplainPlan plan) { - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(scanType, explainPlanAttributes.getExplainScanType()); + assertPlan(plan.getPlanStepsAsAttributes()).table(tableName).iteratorType("PARALLEL 1-WAY") + .scanType(scanType); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson5IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson5IT.java index eea0ca8efe7..0b7d4be1404 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson5IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/Bson5IT.java @@ -20,6 +20,7 @@ import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -43,8 +44,6 @@ import java.util.Properties; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryConstants; @@ -600,6 +599,7 @@ public void testBsonOpsWithSqlConditionsUpdateFailure() throws Exception { } } + @SuppressWarnings("unchecked") private static void testCDCAfterFirstUpsert(Connection conn, String cdcName, Timestamp ts1, Timestamp ts2, BsonDocument bsonDocument1, BsonDocument bsonDocument2, BsonDocument bsonDocument3) throws SQLException, JsonProcessingException { @@ -650,6 +650,7 @@ private static void testCDCAfterFirstUpsert(Connection conn, String cdcName, Tim } } + @SuppressWarnings("unchecked") private static void testCDCPostUpdate(Connection conn, String cdcName, Timestamp ts1, Timestamp ts2, BsonDocument bsonDocument1, BsonDocument bsonDocument2, BsonDocument bsonDocument3) throws SQLException, IOException { @@ -716,6 +717,7 @@ private static void testCDCPostUpdate(Connection conn, String cdcName, Timestamp } } + @SuppressWarnings("unchecked") private static void testCDCUpdateOneRowChange(Connection conn, String cdcName, Timestamp ts1, Timestamp ts2, BsonDocument bsonDocument1) throws SQLException, IOException { try ( @@ -747,11 +749,8 @@ private static void testCDCUpdateOneRowChange(Connection conn, String cdcName, T private static void validateExplainPlan(PreparedStatement ps, String tableName, String scanType) throws SQLException { - ExplainPlan plan = ps.unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(scanType, explainPlanAttributes.getExplainScanType()); + assertPlan(ps.unwrap(PhoenixPreparedStatement.class)).table(tableName) + .iteratorType("PARALLEL 1-WAY").scanType(scanType); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCDefinitionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCDefinitionIT.java index 6941735d238..c2484c5571b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCDefinitionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCDefinitionIT.java @@ -244,8 +244,8 @@ public void testCreateCDCMultitenant() throws Exception { assertEquals(true, indexTable.isMultiTenant()); List idxPkColumns = indexTable.getPKColumns(); assertEquals(":TENANTID", idxPkColumns.get(0).getName().getString()); - assertEquals(": PARTITION_ID()", idxPkColumns.get(1).getName().getString()); - assertEquals(": PHOENIX_ROW_TIMESTAMP()", idxPkColumns.get(2).getName().getString()); + assertEquals(":PARTITION_ID()", idxPkColumns.get(1).getName().getString()); + assertEquals(":PHOENIX_ROW_TIMESTAMP()", idxPkColumns.get(2).getName().getString()); assertEquals(":K", idxPkColumns.get(3).getName().getString()); PTable cdcTable = PhoenixRuntime.getTable(conn, cdcName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCQueryIT.java index 385e4401ec2..6bf2f39f741 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCQueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCQueryIT.java @@ -25,10 +25,12 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT; import static org.apache.phoenix.query.QueryConstants.CDC_EVENT_TYPE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getExplainAttributes; import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.sql.Connection; @@ -53,6 +55,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.filter.DistinctPrefixFilter; import org.apache.phoenix.iterate.ResultIterator; @@ -66,7 +69,6 @@ import org.apache.phoenix.util.CDCUtil; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.ManualEnvironmentEdge; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -147,10 +149,13 @@ public void beforeTest() { private void cdcIndexShouldNotBeUsedForDataTableQueries(Connection conn, String dataTableName, String cdcName) throws Exception { - ResultSet rs = conn.createStatement().executeQuery( - "EXPLAIN SELECT * FROM " + dataTableName + " WHERE PHOENIX_ROW_TIMESTAMP() < CURRENT_TIME()"); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertFalse(explainPlan.contains(cdcName)); + String sql = + "SELECT * FROM " + dataTableName + " WHERE PHOENIX_ROW_TIMESTAMP() < CURRENT_TIME()"; + ExplainPlanAttributes attributes = getExplainAttributes(conn, sql); + String scannedTable = attributes.getTableName(); + assertNotNull(scannedTable); + assertFalse("CDC index " + cdcName + " should not be used, but plan scanned " + scannedTable, + scannedTable.contains(cdcName)); } private boolean isDistinctPrefixFilterIncludedInFilterList(FilterList filterList) { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CastAndCoerceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CastAndCoerceIT.java index 6f80b4b6fb9..1d1203c17b6 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CastAndCoerceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CastAndCoerceIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.ROW1; import static org.apache.phoenix.util.TestUtil.ROW7; import static org.apache.phoenix.util.TestUtil.ROW9; @@ -32,8 +33,6 @@ import java.sql.ResultSet; import java.util.Collection; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.util.PropertiesUtil; import org.junit.Test; @@ -164,12 +163,8 @@ public void testCoerceTinyIntToSmallInt() throws Exception { assertEquals(ROW9, rs.getString(1)); assertFalse(rs.next()); - ExplainPlan plan = - statement.unwrap(PhoenixPreparedStatement.class).optimizeQuery(query).getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(statement.unwrap(PhoenixPreparedStatement.class)).table(tableName) + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN"); } } @@ -187,12 +182,8 @@ public void testCoerceWithRangeScan() throws Exception { assertEquals(ROW9, rs.getString(1)); assertFalse(rs.next()); - ExplainPlan plan = - statement.unwrap(PhoenixPreparedStatement.class).optimizeQuery(query).getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(statement.unwrap(PhoenixPreparedStatement.class)).table(tableName) + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN"); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientHashAggregateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientHashAggregateIT.java index 9bf4caebe7b..ab4333efa72 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientHashAggregateIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ClientHashAggregateIT.java @@ -17,9 +17,12 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getExplainAttributes; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.sql.Connection; @@ -28,8 +31,8 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.Properties; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -113,13 +116,15 @@ private String getQuery(String table, boolean hash, boolean swap, boolean sort) private void verifyExplain(Connection conn, String table, boolean swap, boolean sort) throws Exception { - String query = "EXPLAIN " + getQuery(table, true, swap, sort); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery(query); - String plan = QueryUtil.getExplainPlan(rs); - rs.close(); - assertTrue(plan != null && plan.contains("CLIENT HASH AGGREGATE")); - assertTrue(plan != null && (sort == plan.contains("CLIENT SORTED BY"))); + String query = getQuery(table, true, swap, sort); + ExplainPlanAttributes attributes = getExplainAttributes(conn, query); + assertNotNull(attributes.getClientAggregate()); + assertTrue(attributes.getClientAggregate().contains("CLIENT HASH AGGREGATE")); + if (sort) { + assertNotNull(attributes.getClientSortedBy()); + } else { + assertNull(attributes.getClientSortedBy()); + } } private void verifyResults(Connection conn, String table, int c1, int c2, boolean swap, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CostBasedDecisionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CostBasedDecisionIT.java index 2acfd94ca7d..498488ad7e8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CostBasedDecisionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CostBasedDecisionIT.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertMutationPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -24,16 +26,15 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.util.Map; import java.util.Properties; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; import org.junit.Test; @@ -79,7 +80,9 @@ public void testCostOverridesStaticPlanOrdering1() throws Exception { String query = "SELECT rowkey, c1, c2 FROM " + tableName + " where c1 LIKE 'X0%' ORDER BY rowkey"; // Use the data table plan that opts out order-by when stats are not available. - verifyQueryPlan(query, "FULL SCAN"); + assertPlan(conn, query).scanType("FULL SCAN") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedCount(1).indexRejected( + 0, tableName + "_IDX", OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " (rowkey, c1, c2) VALUES (?, ?, ?)"); @@ -94,7 +97,8 @@ public void testCostOverridesStaticPlanOrdering1() throws Exception { conn.createStatement().execute("UPDATE STATISTICS " + tableName); // Use the index table plan that has a lower cost when stats become available. - verifyQueryPlan(query, "RANGE SCAN"); + assertPlan(conn, query).scanType("RANGE SCAN").indexRule(OptimizerReasons.RULE_COST_BASED) + .indexRejectedNone(); } finally { conn.close(); } @@ -120,9 +124,9 @@ public void testCostOverridesStaticPlanOrdering2() throws Exception { .optimizeQuery().getExplainPlan(); ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertEquals("RANGE SCAN", explainPlanAttributes.getExplainScanType()); assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals(" [*] - ['z']", explainPlanAttributes.getKeyRanges()); + assertEquals("[*] - ['z']", explainPlanAttributes.getKeyRanges()); assertEquals("SERVER AGGREGATE INTO DISTINCT ROWS BY [C1]", explainPlanAttributes.getServerAggregate()); assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); @@ -146,10 +150,11 @@ public void testCostOverridesStaticPlanOrdering2() throws Exception { .getExplainPlan(); explainPlanAttributes = plan.getPlanStepsAsAttributes(); assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertEquals("RANGE SCAN", explainPlanAttributes.getExplainScanType()); assertEquals(indexName + "(" + tableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY AND \"ROWKEY\" <= 'z'", + assertEquals("[1]", explainPlanAttributes.getKeyRanges()); + assertTrue(explainPlanAttributes.isServerFirstKeyOnlyProjection()); + assertEquals("SERVER FILTER BY \"ROWKEY\" <= 'z'", explainPlanAttributes.getServerWhereFilter()); assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"C1\"]", explainPlanAttributes.getServerAggregate()); @@ -182,9 +187,9 @@ public void testCostOverridesStaticPlanOrdering3() throws Exception { .optimizeQuery().getExplainPlan(); ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertEquals("RANGE SCAN", explainPlanAttributes.getExplainScanType()); assertEquals(indexName2 + "(" + tableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [2,*] - [2,9,000]", explainPlanAttributes.getKeyRanges()); + assertEquals("[2,*] - [2,9,000]", explainPlanAttributes.getKeyRanges()); assertEquals( "SERVER FILTER BY ((\"C1\" >= 10 AND \"C1\" <= 20) AND TO_INTEGER(\"C3\") < 5000)", explainPlanAttributes.getServerWhereFilter()); @@ -207,9 +212,9 @@ public void testCostOverridesStaticPlanOrdering3() throws Exception { .getExplainPlan(); explainPlanAttributes = plan.getPlanStepsAsAttributes(); assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertEquals("RANGE SCAN", explainPlanAttributes.getExplainScanType()); assertEquals(indexName1 + "(" + tableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [1,10] - [1,20]", explainPlanAttributes.getKeyRanges()); + assertEquals("[1,10] - [1,20]", explainPlanAttributes.getKeyRanges()); assertEquals("SERVER FILTER BY (\"C2\" < 9000 AND \"C3\" < 5000)", explainPlanAttributes.getServerWhereFilter()); assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); @@ -237,11 +242,13 @@ public void testCostOverridesStaticPlanOrderingInUpsertQuery() throws Exception String query = "UPSERT INTO " + tableName + " SELECT * FROM " + tableName + " where c1 BETWEEN 10 AND 20 AND c2 < 9000 AND C3 < 5000"; // Use the idx2 plan with a wider PK slot span when stats are not available. - verifyQueryPlan(query, - "UPSERT SELECT\n" + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName2 + "(" + tableName - + ")" + " [2,*] - [2,9,000]\n" - + " SERVER FILTER BY ((\"C1\" >= 10 AND \"C1\" <= 20) AND TO_INTEGER(\"C3\") < 5000)\n" - + "CLIENT MERGE SORT"); + assertMutationPlan(conn, query).abstractExplainPlan("UPSERT SELECT").iteratorType("PARALLEL") + .scanType("RANGE SCAN").table(indexName2 + "(" + tableName + ")") + .keyRanges("[2,*] - [2,9,000]") + .serverWhereFilter( + "SERVER FILTER BY ((\"C1\" >= 10 AND \"C1\" <= 20) AND TO_INTEGER(\"C3\") < 5000)") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); PreparedStatement stmt = conn .prepareStatement("UPSERT INTO " + tableName + " (rowkey, c1, c2, c3) VALUES (?, ?, ?, ?)"); @@ -256,10 +263,13 @@ public void testCostOverridesStaticPlanOrderingInUpsertQuery() throws Exception conn.createStatement().execute("UPDATE STATISTICS " + tableName); // Use the idx2 plan that scans less data when stats become available. - verifyQueryPlan(query, - "UPSERT SELECT\n" + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName1 + "(" + tableName - + ")" + " [1,10] - [1,20]\n" + " SERVER FILTER BY (\"C2\" < 9000 AND \"C3\" < 5000)\n" - + "CLIENT MERGE SORT"); + assertMutationPlan(conn, query).abstractExplainPlan("UPSERT SELECT").iteratorType("PARALLEL") + .scanType("RANGE SCAN").table(indexName1 + "(" + tableName + ")") + .keyRanges("[1,10] - [1,20]") + .serverWhereFilter("SERVER FILTER BY (\"C2\" < 9000 AND \"C3\" < 5000)") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_COST_BASED) + .indexRejectedCount(1) + .indexRejected(0, indexName2, OptimizerReasons.REASON_COST_BASED_LOSS); } finally { conn.close(); } @@ -284,11 +294,13 @@ public void testCostOverridesStaticPlanOrderingInDeleteQuery() throws Exception String query = "DELETE FROM " + tableName + " where c1 BETWEEN 10 AND 20 AND c2 < 9000 AND C3 < 5000"; // Use the idx2 plan with a wider PK slot span when stats are not available. - verifyQueryPlan(query, - "DELETE ROWS CLIENT SELECT\n" + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName2 + "(" - + tableName + ")" + " [2,*] - [2,9,000]\n" - + " SERVER FILTER BY ((\"C1\" >= 10 AND \"C1\" <= 20) AND TO_INTEGER(\"C3\") < 5000)\n" - + "CLIENT MERGE SORT"); + assertMutationPlan(conn, query).abstractExplainPlan("DELETE ROWS CLIENT SELECT") + .iteratorType("PARALLEL").scanType("RANGE SCAN").table(indexName2 + "(" + tableName + ")") + .keyRanges("[2,*] - [2,9,000]") + .serverWhereFilter( + "SERVER FILTER BY ((\"C1\" >= 10 AND \"C1\" <= 20) AND TO_INTEGER(\"C3\") < 5000)") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); PreparedStatement stmt = conn .prepareStatement("UPSERT INTO " + tableName + " (rowkey, c1, c2, c3) VALUES (?, ?, ?, ?)"); @@ -303,10 +315,13 @@ public void testCostOverridesStaticPlanOrderingInDeleteQuery() throws Exception conn.createStatement().execute("UPDATE STATISTICS " + tableName); // Use the idx2 plan that scans less data when stats become available. - verifyQueryPlan(query, - "DELETE ROWS CLIENT SELECT\n" + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName1 + "(" - + tableName + ")" + " [1,10] - [1,20]\n" - + " SERVER FILTER BY (\"C2\" < 9000 AND \"C3\" < 5000)\n" + "CLIENT MERGE SORT"); + assertMutationPlan(conn, query).abstractExplainPlan("DELETE ROWS CLIENT SELECT") + .iteratorType("PARALLEL").scanType("RANGE SCAN").table(indexName1 + "(" + tableName + ")") + .keyRanges("[1,10] - [1,20]") + .serverWhereFilter("SERVER FILTER BY (\"C2\" < 9000 AND \"C3\" < 5000)") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_COST_BASED) + .indexRejectedCount(1) + .indexRejected(0, indexName2, OptimizerReasons.REASON_COST_BASED_LOSS); } finally { conn.close(); } @@ -329,12 +344,17 @@ public void testCostOverridesStaticPlanOrderingInUnionQuery() throws Exception { + " where rowkey <= 'z' GROUP BY c1 " + "UNION ALL SELECT c1, max(rowkey), max(c2) FROM " + tableName + " where rowkey >= 'a' GROUP BY c1"; // Use the default plan when stats are not available. - verifyQueryPlan(query, - "UNION ALL OVER 2 QUERIES\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName - + " [*] - ['z']\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [C1]\n" - + " CLIENT MERGE SORT\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName - + " ['a'] - [*]\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY [C1]\n" - + " CLIENT MERGE SORT"); + assertPlan(conn, query).abstractExplainPlan("UNION ALL OVER 2 QUERIES").subPlanCount(2) + .subPlan(0).iteratorType("PARALLEL").scanType("RANGE SCAN").table(tableName) + .keyRanges("[*] - ['z']").serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [C1]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_NO_PK_PREFIX_BOUND).end().subPlan(1) + .iteratorType("PARALLEL").scanType("RANGE SCAN").table(tableName).keyRanges("['a'] - [*]") + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [C1]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_NO_PK_PREFIX_BOUND).end(); PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " (rowkey, c1, c2) VALUES (?, ?, ?)"); @@ -349,16 +369,18 @@ public void testCostOverridesStaticPlanOrderingInUnionQuery() throws Exception { conn.createStatement().execute("UPDATE STATISTICS " + tableName); // Use the optimal plan based on cost when stats become available. - verifyQueryPlan(query, - "UNION ALL OVER 2 QUERIES\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName - + "(" + tableName + ") [1]\n" + " SERVER MERGE [0.C2]\n" - + " SERVER FILTER BY FIRST KEY ONLY AND \"ROWKEY\" <= 'z'\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"C1\"]\n" - + " CLIENT MERGE SORT\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName - + "(" + tableName + ") [1]\n" + " SERVER MERGE [0.C2]\n" - + " SERVER FILTER BY FIRST KEY ONLY AND \"ROWKEY\" >= 'a'\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"C1\"]\n" - + " CLIENT MERGE SORT"); + assertPlan(conn, query).abstractExplainPlan("UNION ALL OVER 2 QUERIES").subPlanCount(2) + .subPlan(0).iteratorType("PARALLEL").scanType("RANGE SCAN") + .table(indexName + "(" + tableName + ")").keyRanges("[1]").serverMergeColumns("[0.C2]") + .serverFirstKeyOnlyProjection(true).serverWhereFilter("SERVER FILTER BY \"ROWKEY\" <= 'z'") + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"C1\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_COST_BASED) + .indexRejectedNone().end().subPlan(1).iteratorType("PARALLEL").scanType("RANGE SCAN") + .table(indexName + "(" + tableName + ")").keyRanges("[1]").serverMergeColumns("[0.C2]") + .serverFirstKeyOnlyProjection(true).serverWhereFilter("SERVER FILTER BY \"ROWKEY\" >= 'a'") + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"C1\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_COST_BASED) + .indexRejectedNone().end(); } finally { conn.close(); } @@ -382,12 +404,16 @@ public void testCostOverridesStaticPlanOrderingInJoinQuery() throws Exception { + " where rowkey <= 'z' GROUP BY c1) t2 " + "ON t1.rowkey = t2.mrk WHERE t1.c1 LIKE 'X0%' ORDER BY t1.rowkey"; // Use the default plan when stats are not available. - verifyQueryPlan(query, - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName + "\n" - + " SERVER FILTER BY C1 LIKE 'X0%'\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " [*] - ['z']\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [C1]\n" - + " CLIENT MERGE SORT\n" + " DYNAMIC SERVER FILTER BY T1.ROWKEY IN (T2.MRK)"); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(tableName) + .serverWhereFilter("SERVER FILTER BY C1 LIKE 'X0%'") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY T1.ROWKEY IN (T2.MRK)").indexRule(null) + .indexRejectedNone().subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(tableName) + .keyRanges("[*] - ['z']").serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [C1]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_NO_PK_PREFIX_BOUND); PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " (rowkey, c1, c2) VALUES (?, ?, ?)"); @@ -402,17 +428,19 @@ public void testCostOverridesStaticPlanOrderingInJoinQuery() throws Exception { conn.createStatement().execute("UPDATE STATISTICS " + tableName); // Use the optimal plan based on cost when stats become available. - verifyQueryPlan(query, - "CLIENT PARALLEL 626-WAY RANGE SCAN OVER " + indexName + "(" + tableName - + ") [1,'X0'] - [1,'X1']\n" + " SERVER MERGE [0.C2]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER SORTED BY [\"T1.:ROWKEY\"]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName + "(" + tableName - + ") [1]\n" + " SERVER MERGE [0.C2]\n" - + " SERVER FILTER BY FIRST KEY ONLY AND \"ROWKEY\" <= 'z'\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"C1\"]\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"T1.:ROWKEY\" IN (T2.MRK)"); + assertPlan(conn, query).iteratorType("PARALLEL 626-WAY").scanType("RANGE SCAN") + .table(indexName + "(" + tableName + ")").keyRanges("[1,'X0'] - [1,'X1']") + .serverMergeColumns("[0.C2]").serverFirstKeyOnlyProjection(true) + .serverSortedBy("[\"T1.:ROWKEY\"]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"T1.:ROWKEY\" IN (T2.MRK)").indexRule(null) + .indexRejectedNone().subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(indexName + "(" + tableName + ")").keyRanges("[1]").serverMergeColumns("[0.C2]") + .serverFirstKeyOnlyProjection(true).serverWhereFilter("SERVER FILTER BY \"ROWKEY\" <= 'z'") + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"C1\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_COST_BASED) + .indexRejectedNone(); } finally { conn.close(); } @@ -435,8 +463,7 @@ public void testHintOverridesCost() throws Exception { String hintedQuery = query.replaceFirst("SELECT", "SELECT /*+ INDEX(" + tableName + " " + tableName + "_idx) */"); String dataPlan = "[C1]"; - String indexPlan = - "SERVER FILTER BY FIRST KEY ONLY AND (\"ROWKEY\" >= 1 AND \"ROWKEY\" <= 10)"; + String indexPlan = "SERVER FILTER BY (\"ROWKEY\" >= 1 AND \"ROWKEY\" <= 10)"; // Use the index table plan that opts out order-by when stats are not available. ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) @@ -479,17 +506,9 @@ public void testJoinStrategy() throws Exception { + "ON t1.ID = t2.ID"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { - ExplainPlan plan = conn.prepareStatement(q).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("SORT-MERGE-JOIN (INNER)", explainPlanAttributes.getAbstractExplainPlan()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(testTable500, explainPlanAttributes.getTableName()); - ExplainPlanAttributes rhsTable = explainPlanAttributes.getRhsJoinQueryExplainPlan(); - assertEquals("PARALLEL 1-WAY", rhsTable.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", rhsTable.getExplainScanType()); - assertEquals(testTable1000, rhsTable.getTableName()); + assertPlan(conn, q).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").lhs() + .iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable500).end().rhs() + .iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable1000); } } @@ -502,20 +521,11 @@ public void testJoinStrategy2() throws Exception { + "ON t1.ID = t2.ID\n" + "WHERE t1.COL1 < 200"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { - ExplainPlan plan = conn.prepareStatement(q).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("SORT-MERGE-JOIN (INNER)", explainPlanAttributes.getAbstractExplainPlan()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("SERVER FILTER BY COL1 < 200", explainPlanAttributes.getServerWhereFilter()); - assertEquals(testTable500, explainPlanAttributes.getTableName()); - assertEquals("CLIENT AGGREGATE INTO SINGLE ROW", explainPlanAttributes.getClientAggregate()); - ExplainPlanAttributes rhsTable = explainPlanAttributes.getRhsJoinQueryExplainPlan(); - assertEquals("PARALLEL 1-WAY", rhsTable.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", rhsTable.getExplainScanType()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", rhsTable.getServerWhereFilter()); - assertEquals(testTable1000, rhsTable.getTableName()); + assertPlan(conn, q).abstractExplainPlan("SORT-MERGE-JOIN (INNER)") + .clientAggregate("CLIENT AGGREGATE INTO SINGLE ROW").lhs().iteratorType("PARALLEL 1-WAY") + .scanType("FULL SCAN").table(testTable500).serverWhereFilter("SERVER FILTER BY COL1 < 200") + .end().rhs().iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable1000) + .serverFirstKeyOnlyProjection(true); } } @@ -524,10 +534,14 @@ public void testJoinStrategy2() throws Exception { public void testJoinStrategy3() throws Exception { String q = "SELECT *\n" + "FROM " + testTable500 + " t1 JOIN " + testTable1000 + " t2\n" + "ON t1.COL1 = t2.ID\n" + "WHERE t1.ID > 200"; - String expected = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + testTable1000 + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + testTable500 + " [201] - [*]\n" + " DYNAMIC SERVER FILTER BY T2.ID IN (T1.COL1)"; - verifyQueryPlan(q, expected); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable1000) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY T2.ID IN (T1.COL1)").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(testTable500) + .keyRanges("[201] - [*]"); + } } /** @@ -538,10 +552,13 @@ public void testJoinStrategy3() throws Exception { public void testJoinStrategy4() throws Exception { String q = "SELECT *\n" + "FROM " + testTable990 + " t1 JOIN " + testTable1000 + " t2\n" + "ON t1.ID = t2.COL1"; - String expected = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + testTable990 + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + testTable1000 + "\n" + " DYNAMIC SERVER FILTER BY T1.ID IN (T2.COL1)"; - verifyQueryPlan(q, expected); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable990) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY T1.ID IN (T2.COL1)").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable1000); + } } /** Hash-join wins over sort-merge-join w/ smaller side ordered. */ @@ -549,10 +566,14 @@ public void testJoinStrategy4() throws Exception { public void testJoinStrategy5() throws Exception { String q = "SELECT *\n" + "FROM " + testTable500 + " t1 JOIN " + testTable1000 + " t2\n" + "ON t1.ID = t2.COL1\n" + "WHERE t1.ID > 200"; - String expected = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + testTable1000 + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + testTable500 + " [201] - [*]"; - verifyQueryPlan(q, expected); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable1000) + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(testTable500) + .keyRanges("[201] - [*]"); + } } /** Hash-join wins over sort-merge-join w/o any side ordered. */ @@ -560,10 +581,14 @@ public void testJoinStrategy5() throws Exception { public void testJoinStrategy6() throws Exception { String q = "SELECT *\n" + "FROM " + testTable500 + " t1 JOIN " + testTable1000 + " t2\n" + "ON t1.COL1 = t2.COL1\n" + "WHERE t1.ID > 200"; - String expected = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + testTable1000 + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + testTable500 + " [201] - [*]"; - verifyQueryPlan(q, expected); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable1000) + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(testTable500) + .keyRanges("[201] - [*]"); + } } /** @@ -575,11 +600,14 @@ public void testJoinStrategy6() throws Exception { public void testJoinStrategy7() throws Exception { String q = "SELECT *\n" + "FROM " + testTable500 + " t1 JOIN " + testTable1000 + " t2\n" + "ON t1.ID = t2.ID\n" + "ORDER BY t1.COL1"; - String expected = "CLIENT PARALLEL 1001-WAY FULL SCAN OVER " + testTable1000 + "\n" - + " SERVER SORTED BY [T1.COL1]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + testTable500 + "\n" + " DYNAMIC SERVER FILTER BY T2.ID IN (T1.ID)"; - verifyQueryPlan(q, expected); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).iteratorType("PARALLEL 1001-WAY").scanType("FULL SCAN") + .table(testTable1000).serverSortedBy("[T1.COL1]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY T2.ID IN (T1.ID)").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable500); + } } /** @@ -593,19 +621,10 @@ public void testJoinStrategy8() throws Exception { + "ON t1.ID = t2.ID\n" + "ORDER BY t1.COL1 LIMIT 5"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { - ExplainPlan plan = conn.prepareStatement(q).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("SORT-MERGE-JOIN (INNER)", explainPlanAttributes.getAbstractExplainPlan()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(testTable500, explainPlanAttributes.getTableName()); - assertEquals("[T1.COL1]", explainPlanAttributes.getClientSortedBy()); - assertEquals(new Integer(5), explainPlanAttributes.getClientRowLimit()); - ExplainPlanAttributes rhsTable = explainPlanAttributes.getRhsJoinQueryExplainPlan(); - assertEquals("PARALLEL 1-WAY", rhsTable.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", rhsTable.getExplainScanType()); - assertEquals(testTable1000, rhsTable.getTableName()); + assertPlan(conn, q).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").clientSortedBy("[T1.COL1]") + .clientRowLimit(5).lhs().iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(testTable500).end().rhs().iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(testTable1000); } } @@ -617,11 +636,15 @@ public void testJoinStrategy9() throws Exception { String q = "SELECT *\n" + "FROM " + testTable1000 + " t1 LEFT JOIN " + testTable500 + " t2\n" + "ON t1.ID = t2.ID AND t2.ID > 200\n" + "LEFT JOIN " + testTable990 + " t3\n" + "ON t1.ID = t3.ID AND t3.ID < 100"; - String expected = "SORT-MERGE-JOIN (LEFT) TABLES\n" + " SORT-MERGE-JOIN (LEFT) TABLES\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + testTable1000 + "\n" + " AND\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + testTable500 + " [201] - [*]\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + testTable990 + " [*] - [100]"; - verifyQueryPlan(q, expected); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).abstractExplainPlan("SORT-MERGE-JOIN (LEFT)").lhs() + .abstractExplainPlan("SORT-MERGE-JOIN (LEFT)").lhs().iteratorType("PARALLEL 1-WAY") + .scanType("FULL SCAN").table(testTable1000).end().rhs().iteratorType("PARALLEL 1-WAY") + .scanType("RANGE SCAN").table(testTable500).keyRanges("[201] - [*]").end().end().rhs() + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(testTable990) + .keyRanges("[*] - [100]"); + } } /** @@ -632,13 +655,16 @@ public void testJoinStrategy10() throws Exception { String q = "SELECT *\n" + "FROM " + testTable1000 + " t1 JOIN " + testTable500 + " t2\n" + "ON t1.ID = t2.COL1 AND t2.ID > 200\n" + "JOIN " + testTable990 + " t3\n" + "ON t1.ID = t3.ID AND t3.ID < 100"; - String expected = - "SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + testTable1000 + "\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + testTable500 + " [201] - [*]\n" - + " DYNAMIC SERVER FILTER BY T1.ID IN (T2.COL1)\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + testTable990 + " [*] - [100]"; - verifyQueryPlan(q, expected); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").lhs() + .iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable1000) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY T1.ID IN (T2.COL1)").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(testTable500) + .keyRanges("[201] - [*]").end().end().rhs().iteratorType("PARALLEL 1-WAY") + .scanType("RANGE SCAN").table(testTable990).keyRanges("[*] - [100]"); + } } /** @@ -650,11 +676,17 @@ public void testJoinStrategy11() throws Exception { String q = "SELECT *\n" + "FROM " + testTable1000 + " t1 JOIN " + testTable500 + " t2\n" + "ON t1.COL2 = t2.COL1 AND t2.ID > 200\n" + "JOIN " + testTable990 + " t3\n" + "ON t1.COL1 = t3.COL2 AND t3.ID < 100"; - String expected = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + testTable1000 + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + testTable500 + " [201] - [*]\n" + " PARALLEL INNER-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + testTable990 + " [*] - [100]"; - verifyQueryPlan(q, expected); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable1000) + .subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(testTable500) + .keyRanges("[201] - [*]").end().subPlan(1) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT */") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(testTable990) + .keyRanges("[*] - [100]"); + } } /** @@ -665,22 +697,16 @@ public void testJoinStrategy11() throws Exception { public void testJoinStrategy12() throws Exception { String q = "SELECT *\n" + "FROM " + testTable1000 + " t1 JOIN " + testTable990 + " t2\n" + "ON t1.COL2 = t2.COL1\n" + "JOIN " + testTable990 + " t3\n" + "ON t1.COL1 = t3.COL2"; - String expected = - "SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1001-WAY FULL SCAN OVER " - + testTable1000 + "\n" + " SERVER SORTED BY [T1.COL1]\n" + " CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + testTable990 + "\n" + "AND\n" - + " CLIENT PARALLEL 991-WAY FULL SCAN OVER " + testTable990 + "\n" - + " SERVER SORTED BY [T3.COL2]\n" + " CLIENT MERGE SORT"; - verifyQueryPlan(q, expected); - } - - private static void verifyQueryPlan(String query, String expected) throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - Connection conn = DriverManager.getConnection(getUrl(), props); - ResultSet rs = conn.createStatement().executeQuery("explain " + query); - String plan = QueryUtil.getExplainPlan(rs); - assertTrue("Expected '" + expected + "' in the plan:\n" + plan + ".", plan.contains(expected)); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + assertPlan(conn, q).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").lhs() + .iteratorType("PARALLEL 1001-WAY").scanType("FULL SCAN").table(testTable1000) + .serverSortedBy("[T1.COL1]").clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(testTable990).end().end().rhs() + .iteratorType("PARALLEL 991-WAY").scanType("FULL SCAN").table(testTable990) + .serverSortedBy("[T3.COL2]").clientSortAlgo("CLIENT MERGE SORT"); + } } private static String initTestTableValues(int rows) throws Exception { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CountDistinctApproximateHyperLogLogIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CountDistinctApproximateHyperLogLogIT.java index 193e0bf0fcc..8b4dc54e01a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CountDistinctApproximateHyperLogLogIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CountDistinctApproximateHyperLogLogIT.java @@ -17,14 +17,12 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.*; import java.sql.*; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.schema.ColumnNotFoundException; import org.apache.phoenix.util.PropertiesUtil; import org.junit.Before; @@ -124,14 +122,8 @@ public void testDistinctCountPlanExplain() throws Exception { String query = "SELECT APPROX_COUNT_DISTINCT(i1||i2) FROM " + tableName; try (Connection conn = DriverManager.getConnection(getUrl(), props)) { prepareTableWithValues(conn, 100); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER AGGREGATE INTO SINGLE ROW", explainPlanAttributes.getServerAggregate()); + assertPlan(conn, query).table(tableName).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .serverFirstKeyOnlyProjection(true).serverAggregate("SERVER AGGREGATE INTO SINGLE ROW"); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java index f4f57bafc8a..c196118b78f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java @@ -19,6 +19,7 @@ import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY; import static org.apache.phoenix.mapreduce.index.IndexUpgradeTool.ROLLBACK_OP; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -83,7 +84,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -1072,9 +1072,7 @@ public void testCreateTableWithNamespaceMappingEnabled() throws Exception { conn.createStatement() .execute("CREATE TABLE " + table + " (PK VARCHAR PRIMARY KEY, " + CF + ".COL VARCHAR)"); - assertTrue(QueryUtil - .getExplainPlan(conn.createStatement().executeQuery("explain select * from " + table)) - .contains(NS + ":" + TBL)); + assertPlan(conn, "select * from " + table).tableContains(NS + ":" + TBL); conn.createStatement().execute("DROP TABLE " + table); } @@ -1085,9 +1083,7 @@ public void testCreateTableWithNamespaceMappingEnabled() throws Exception { conn.createStatement() .execute("CREATE TABLE " + table + " (PK VARCHAR PRIMARY KEY, " + CF + ".COL VARCHAR)"); - assertTrue(QueryUtil - .getExplainPlan(conn.createStatement().executeQuery("explain select * from " + table)) - .contains(NS + "." + TBL)); + assertPlan(conn, "select * from " + table).tableContains(NS + "." + TBL); conn.createStatement().execute("DROP TABLE " + table); } @@ -1098,9 +1094,7 @@ public void testCreateTableWithNamespaceMappingEnabled() throws Exception { conn.createStatement() .execute("CREATE TABLE " + table + " (PK VARCHAR PRIMARY KEY, " + CF + ".COL VARCHAR)"); - assertTrue(QueryUtil - .getExplainPlan(conn.createStatement().executeQuery("explain select * from " + table)) - .contains(NS + ":" + NS + "." + TBL)); + assertPlan(conn, "select * from " + table).tableContains(NS + ":" + NS + "." + TBL); conn.createStatement().execute("DROP TABLE " + table); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java index abd6a212e9e..5fa004dd402 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_ATTRIB; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -45,7 +46,6 @@ import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.util.DateUtil; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -521,8 +521,7 @@ public void testImportWithDifferentPhysicalName() throws Exception { assertEquals("FirstName 2", rs.getString(2)); String selectFromIndex = "SELECT FIRST_NAME FROM " + fullTableName + " where FIRST_NAME='FirstName 1'"; - rs = stmt.executeQuery("EXPLAIN " + selectFromIndex); - assertTrue(QueryUtil.getExplainPlan(rs).contains(indexTableName)); + assertPlan(conn, selectFromIndex).tableContains(indexTableName); rs = stmt.executeQuery(selectFromIndex); assertTrue(rs.next()); assertEquals("FirstName 1", rs.getString(1)); @@ -538,8 +537,7 @@ public void testImportWithDifferentPhysicalName() throws Exception { "--index-table", indexTableName, "--zookeeper", zkQuorum, "--corruptindexes" }); assertEquals(0, exitCode); selectFromIndex = "SELECT FIRST_NAME FROM " + fullTableName + " where FIRST_NAME='FirstName 3'"; - rs = stmt.executeQuery("EXPLAIN " + selectFromIndex); - assertTrue(QueryUtil.getExplainPlan(rs).contains(indexTableName)); + assertPlan(conn, selectFromIndex).tableContains(indexTableName); rs = stmt.executeQuery(selectFromIndex); assertTrue(rs.next()); assertEquals("FirstName 3", rs.getString(1)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java index c2a31842f9b..a23db7c63fd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertMutationPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -52,16 +54,17 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.DeleteCompiler; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.MutationPlan; import org.apache.phoenix.end2end.index.IndexTestUtil; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.DeleteStatement; import org.apache.phoenix.parse.SQLParser; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -192,20 +195,27 @@ private static void assertIndexUsed(Connection conn, String query, String indexN private static void assertIndexUsed(Connection conn, String query, List binds, String indexName, boolean expectedToBeUsed, boolean local) throws SQLException { - PreparedStatement stmt = conn.prepareStatement("EXPLAIN " + query); + PhoenixPreparedStatement stmt = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class); for (int i = 0; i < binds.size(); i++) { stmt.setObject(i + 1, binds.get(i)); } - ResultSet rs = stmt.executeQuery(); - String explainPlan = QueryUtil.getExplainPlan(rs); - // It's very difficult currently to check if a local index is being used - // This check is brittle as it checks that the index ID appears in the range scan - // TODO: surface QueryPlan from MutationPlan + boolean isMutation = query.trim().toUpperCase().startsWith("DELETE") + || query.trim().toUpperCase().startsWith("UPSERT"); + ExplainPlanAttributes attributes = + (isMutation ? stmt.compileMutation().getExplainPlan() : stmt.optimizeQuery().getExplainPlan()) + .getPlanStepsAsAttributes(); + // It's very difficult currently to check if a local index is being used. + // This check is brittle as it checks that the index ID appears in the range scan. if (local) { - assertEquals(expectedToBeUsed, - explainPlan.contains(indexName + " [1]") || explainPlan.contains(indexName + " [1,")); + String keyRanges = attributes.getKeyRanges(); + boolean used = indexName.equals(attributes.getTableName()) && keyRanges != null + && (keyRanges.startsWith("[1]") || keyRanges.startsWith("[1,")); + assertEquals(expectedToBeUsed, used); + } else if (expectedToBeUsed) { + assertPlan(attributes).table(indexName); } else { - assertEquals(expectedToBeUsed, explainPlan.contains(" SCAN OVER " + indexName)); + assertNotEquals(indexName, attributes.getTableName()); } } @@ -499,16 +509,18 @@ public void testPointDeleteRowFromTableWithImmutableIndex(boolean localIndex, if (!autoCommit) { con.commit(); } - psDelete = con.prepareStatement("EXPLAIN " + dml); - psDelete.setString(1, "AA"); - psDelete.setString(2, "BB"); - psDelete.setString(3, "CC"); - psDelete.setDate(4, date); - String explainPlan = QueryUtil.getExplainPlan(psDelete.executeQuery()); + PhoenixPreparedStatement explainStmt = + con.prepareStatement(dml).unwrap(PhoenixPreparedStatement.class); + explainStmt.setString(1, "AA"); + explainStmt.setString(2, "BB"); + explainStmt.setString(3, "CC"); + explainStmt.setDate(4, date); + ExplainPlanAttributes attributes = + explainStmt.compileMutation().getExplainPlan().getPlanStepsAsAttributes(); if (addNonPKIndex) { - assertNotEquals("DELETE SINGLE ROW", explainPlan); + assertNotEquals("DELETE SINGLE ROW", attributes.getAbstractExplainPlan()); } else { - assertEquals("DELETE SINGLE ROW", explainPlan); + assertPlan(attributes).abstractExplainPlan("DELETE SINGLE ROW"); } assertDeleted(con, tableName, indexName1, indexName2, indexName3); @@ -927,26 +939,20 @@ public void testDeleteFilterWithMultipleIndexes() throws Exception { } try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.setAutoCommit(true); - try (Statement statement = conn.createStatement()) { - ResultSet rs = statement.executeQuery("EXPLAIN " + delete); - String explainPlan = QueryUtil.getExplainPlan(rs); - // Verify index is used for the delete query - IndexToolIT.assertExplainPlan(false, explainPlan, tableName, indexName1); - } + // Verify index is used for the delete query + assertMutationPlan(conn, delete).scanType("RANGE SCAN").table(indexName1); // Created the second index try (Statement statement = conn.createStatement()) { statement.execute(indexDdl2); } + // Verify index is used for the delete query + assertMutationPlan(conn, delete).scanType("RANGE SCAN").table(indexName1); try (Statement statement = conn.createStatement()) { - ResultSet rs = statement.executeQuery("EXPLAIN " + delete); - String explainPlan = QueryUtil.getExplainPlan(rs); - // Verify index is used for the delete query - IndexToolIT.assertExplainPlan(false, explainPlan, tableName, indexName1); statement.executeUpdate(delete); // Count the number of rows String query = "SELECT COUNT(*) from " + tableName; // There should be no rows on the data table - rs = conn.createStatement().executeQuery(query); + ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(0, rs.getInt(1)); query = "SELECT COUNT(*) from " + indexName1; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java index d6cde17cf02..85ac718ac61 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.A_VALUE; import static org.apache.phoenix.util.TestUtil.B_VALUE; import static org.apache.phoenix.util.TestUtil.C_VALUE; @@ -40,11 +41,11 @@ import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Collection; import java.util.List; import java.util.Properties; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -54,8 +55,6 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; @@ -68,12 +67,10 @@ public class DerivedTableIT extends ParallelStatsDisabledIT { public TestName name = new TestName(); private String[] indexDDL; - private String[] plans; + private PlanSpec[] plans; private String tableName; - private static final Logger LOGGER = LoggerFactory.getLogger(DerivedTableIT.class); - - public DerivedTableIT(String[] indexDDL, String[] plans) { + public DerivedTableIT(String[] indexDDL, PlanSpec[] plans) { this.indexDDL = indexDDL; this.plans = plans; } @@ -92,13 +89,6 @@ public void initTable() throws Exception { conn.createStatement().execute(ddl); } } - String[] newplan = new String[plans.length]; - if (plans != null && plans.length > 0) { - for (int i = 0; i < plans.length; i++) { - newplan[i] = plans[i].replace(dynamicTableName, tableName); - } - plans = newplan; - } } @After @@ -108,31 +98,53 @@ public void cleanUp() throws Exception { assertFalse("refCount leaked", refCountLeaked); } + /** Structured EXPLAIN plan expectations for a derived-table query */ + private static final class PlanSpec { + final String iteratorType; + final String tableSuffix; + final String serverAggregate; + final String[] clientSteps; + + PlanSpec(String iteratorType, String tableSuffix, String serverAggregate, + String... clientSteps) { + this.iteratorType = iteratorType; + this.tableSuffix = tableSuffix; + this.serverAggregate = serverAggregate; + this.clientSteps = clientSteps; + } + } + + private void verifyPlan(Connection conn, String query, PlanSpec spec) throws SQLException { + assertPlan(conn, query).iteratorType(spec.iteratorType).scanType("FULL SCAN") + .table(tableName + spec.tableSuffix).serverAggregate(spec.serverAggregate) + .clientSteps(spec.clientSteps); + } + @Parameters(name = "DerivedTableIT_{index}") // name is used by failsafe as file name in reports public static synchronized Collection data() { List testCases = Lists.newArrayList(); - testCases.add(new String[][] { - { "CREATE INDEX " + dynamicTableName + "_DERIVED_IDX ON " + dynamicTableName + testCases.add(new Object[] { + new String[] { "CREATE INDEX " + dynamicTableName + "_DERIVED_IDX ON " + dynamicTableName + " (a_byte) INCLUDE (A_STRING, B_STRING)" }, - { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dynamicTableName + "_DERIVED_IDX \n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" - + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [\"B_STRING\"]\n" + "CLIENT SORTED BY [A]\n" - + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dynamicTableName + "_DERIVED_IDX \n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" - + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY [A]\n" - + "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]\n" + "CLIENT SORTED BY [A DESC]" } }); - testCases.add(new String[][] { {}, - { "CLIENT PARALLEL 4-WAY FULL SCAN OVER " + dynamicTableName + " \n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" - + "CLIENT MERGE SORT\n" + "CLIENT SORTED BY [B_STRING]\n" + "CLIENT SORTED BY [A]\n" - + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" + "CLIENT SORTED BY [A DESC]", - - "CLIENT PARALLEL 4-WAY FULL SCAN OVER " + dynamicTableName + " \n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" - + "CLIENT MERGE SORT\n" + "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY [A]\n" - + "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]\n" + "CLIENT SORTED BY [A DESC]" } }); + new PlanSpec[] { + new PlanSpec("PARALLEL 1-WAY", "_DERIVED_IDX", + "SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]", + "CLIENT MERGE SORT", "CLIENT SORTED BY [\"B_STRING\"]", "CLIENT SORTED BY [A]", + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]", "CLIENT SORTED BY [A DESC]"), + new PlanSpec("PARALLEL 1-WAY", "_DERIVED_IDX", + "SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]", + "CLIENT MERGE SORT", "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY [A]", + "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]", "CLIENT SORTED BY [A DESC]") } }); + testCases.add(new Object[] { new String[] {}, + new PlanSpec[] { + new PlanSpec("PARALLEL 4-WAY", "", + "SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]", "CLIENT MERGE SORT", + "CLIENT SORTED BY [B_STRING]", "CLIENT SORTED BY [A]", + "CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]", "CLIENT SORTED BY [A DESC]"), + new PlanSpec("PARALLEL 4-WAY", "", + "SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]", "CLIENT MERGE SORT", + "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY [A]", + "CLIENT DISTINCT ON [COLLECTDISTINCT(B)]", "CLIENT SORTED BY [A DESC]") } }); return testCases; } @@ -384,12 +396,7 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query); - String explainPlanOutput = QueryUtil.getExplainPlan(rs); - LOGGER.info("Explain plan output: {}", explainPlanOutput); - String[] splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); - String[] secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); - assertEquals(plans[0], splitExplainPlan[0] + secondSplitExplainPlan[1]); + verifyPlan(conn, query, plans[0]); // distinct b (groupby a, b) groupby a orderby a query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM " @@ -411,12 +418,7 @@ public void testDerivedTableWithGroupBy() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query); - explainPlanOutput = QueryUtil.getExplainPlan(rs); - LOGGER.info("Explain plan output: {}", explainPlanOutput); - splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region="); - secondSplitExplainPlan = splitExplainPlan[1].split("]\\)"); - assertEquals(plans[1], splitExplainPlan[0] + secondSplitExplainPlan[1]); + verifyPlan(conn, query, plans[1]); // (orderby) groupby query = "SELECT t.a_string, count(*) FROM (SELECT * FROM " + tableName diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java index bebf007fb86..e1eedac0e77 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java @@ -17,10 +17,13 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getExplainAttributes; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.sql.Connection; @@ -29,8 +32,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Properties; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -232,8 +235,12 @@ private void testCommonPlans(String testTable, String contains) throws Exception } private void testPlan(String query, boolean optimizable) throws Exception { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(optimizable, QueryUtil.getExplainPlan(rs).contains(PREFIX)); + ExplainPlanAttributes attributes = getExplainAttributes(conn, query); + if (optimizable) { + assertNotNull(attributes.getServerDistinctFilter()); + } else { + assertNull(attributes.getServerDistinctFilter()); + } } @Test diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/EmptyColumnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/EmptyColumnIT.java index 5e0033bb4f9..d52749615d1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/EmptyColumnIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/EmptyColumnIT.java @@ -29,10 +29,12 @@ import static org.apache.phoenix.query.PhoenixTestBuilder.SchemaBuilder.TableOptions; import static org.apache.phoenix.query.PhoenixTestBuilder.SchemaBuilder.TenantViewIndexOptions; import static org.apache.phoenix.query.PhoenixTestBuilder.SchemaBuilder.TenantViewOptions; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getExplainAttributes; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; import java.io.IOException; import java.sql.Connection; @@ -50,7 +52,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.jdbc.PhoenixResultSet; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.query.PhoenixTestBuilder.BasicDataWriter; import org.apache.phoenix.query.PhoenixTestBuilder.DataSupplier; import org.apache.phoenix.query.PhoenixTestBuilder.DataWriter; @@ -60,7 +62,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.ManualEnvironmentEdge; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Ignore; @@ -746,11 +747,10 @@ public void testMaskingWithDistinctPrefixFilter() throws Exception { injectEdge.setValue(EnvironmentEdgeManager.currentTimeMillis() + ttl * 1000 + 2); EnvironmentEdgeManager.injectEdge(injectEdge); String distinctQuery = "SELECT DISTINCT id1 FROM " + dataTableName; + ExplainPlanAttributes attributes = getExplainAttributes(conn, distinctQuery); + assertPlan(attributes).serverEmptyColumnOnlyProjection(true); + assertNotNull(attributes.getServerDistinctFilter()); try (ResultSet rs = conn.createStatement().executeQuery(distinctQuery)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains("SERVER FILTER BY EMPTY COLUMN ONLY")); - assertTrue(explainPlan.contains("SERVER DISTINCT PREFIX FILTER OVER")); // all the rows should have been masked assertFalse(rs.next()); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsDisabledIT.java index 2565a3ba80d..948f81f2a37 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsDisabledIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsDisabledIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.end2end.ExplainPlanWithStatsEnabledIT.getByteRowEstimates; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -27,14 +28,10 @@ import java.sql.ResultSet; import java.util.List; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ExplainPlanWithStatsEnabledIT.Estimate; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -256,12 +253,10 @@ public void testDescTimestampAtBoundary() throws Exception { + " ) IMMUTABLE_ROWS=true\n" + " ,SALT_BUCKETS=20"); String query = "select * from foo where a = 'a' and b >= timestamp '2016-01-28 00:00:00' and b < timestamp '2016-01-29 00:00:00'"; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 20-WAY RANGE SCAN OVER FOO [X'00','a',~'2016-01-28 23:59:59.999'] - [X'13','a',~'2016-01-28 00:00:00.000']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - queryPlan); + assertPlan(conn, query).scanType("RANGE SCAN").table("FOO") + .keyRanges( + "[X'00','a',~'2016-01-28 23:59:59.999'] -" + " [X'13','a',~'2016-01-28 00:00:00.000']") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT"); } } @@ -278,11 +273,10 @@ public void testUseOfRoundRobinIteratorSurfaced() throws Exception { + " ) IMMUTABLE_ROWS=true\n" + " ,SALT_BUCKETS=20"); String query = "select * from " + tableName + " where a = 'a' and b >= timestamp '2016-01-28 00:00:00' and b < timestamp '2016-01-29 00:00:00'"; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 20-WAY ROUND ROBIN RANGE SCAN OVER " + tableName - + " [X'00','a',~'2016-01-28 23:59:59.999'] - [X'13','a',~'2016-01-28 00:00:00.000']\n" - + " SERVER FILTER BY FIRST KEY ONLY", queryPlan); + assertPlan(conn, query).useRoundRobinIterator(true).scanType("RANGE SCAN").table(tableName) + .keyRanges( + "[X'00','a',~'2016-01-28 23:59:59.999'] -" + " [X'13','a',~'2016-01-28 00:00:00.000']") + .serverFirstKeyOnlyProjection(true); } } @@ -346,61 +340,33 @@ public void testRangeScanWithMetadataLookup() throws Exception { ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(53, rs.getInt(1)); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " [*] - ['b']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO SINGLE ROW", - queryPlan); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(2, planAttributes.getNumRegionLocationLookups()); + assertPlan(conn, query).scanType("RANGE SCAN").table(tableName).keyRanges("[*] - ['b']") + .serverFirstKeyOnlyProjection(true).serverAggregate("SERVER AGGREGATE INTO SINGLE ROW") + .numRegionLocationLookups(2); query = "select count(*) from " + tableName + " where PK1 <= 'cd'"; rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(128, rs.getInt(1)); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " [*] - ['cd']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO SINGLE ROW", - queryPlan); - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(3, planAttributes.getNumRegionLocationLookups()); + assertPlan(conn, query).scanType("RANGE SCAN").table(tableName).keyRanges("[*] - ['cd']") + .serverFirstKeyOnlyProjection(true).serverAggregate("SERVER AGGREGATE INTO SINGLE ROW") + .numRegionLocationLookups(3); query = "select count(*) from " + tableName + " where PK1 LIKE 'ef%'"; rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(25, rs.getInt(1)); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " ['ef'] - ['eg']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO SINGLE ROW", - queryPlan); - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(1, planAttributes.getNumRegionLocationLookups()); + assertPlan(conn, query).scanType("RANGE SCAN").table(tableName).keyRanges("['ef'] - ['eg']") + .serverFirstKeyOnlyProjection(true).serverAggregate("SERVER AGGREGATE INTO SINGLE ROW") + .numRegionLocationLookups(1); query = "select count(*) from " + tableName + " where PK1 > 'de'"; rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(75, rs.getInt(1)); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " ['de'] - [*]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO SINGLE ROW", - queryPlan); - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(1, planAttributes.getNumRegionLocationLookups()); + assertPlan(conn, query).scanType("RANGE SCAN").table(tableName).keyRanges("['de'] - [*]") + .serverFirstKeyOnlyProjection(true).serverAggregate("SERVER AGGREGATE INTO SINGLE ROW") + .numRegionLocationLookups(1); } } @@ -494,29 +460,15 @@ public void testMultiTenantWithMetadataLookup() throws Exception { assertTrue(rs.next()); assertEquals(50, rs.getInt(1)); - rs = tenantConn.createStatement().executeQuery("EXPLAIN " + query); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " ['ab12']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO SINGLE ROW", - queryPlan); - ExplainPlan plan = tenantConn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(1, planAttributes.getNumRegionLocationLookups()); + assertPlan(tenantConn, query).scanType("RANGE SCAN").table(tableName).keyRanges("['ab12']") + .serverFirstKeyOnlyProjection(true).serverAggregate("SERVER AGGREGATE INTO SINGLE ROW") + .numRegionLocationLookups(1); } try (Connection tenantConn = getTenantConnection("cd12")) { String query = "select * from " + view03 + " order by col2"; - - ResultSet rs = tenantConn.createStatement().executeQuery("EXPLAIN " + query); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " ['cd12']\n" - + " SERVER SORTED BY [COL2]\n" + "CLIENT MERGE SORT", queryPlan); - ExplainPlan plan = tenantConn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(1, planAttributes.getNumRegionLocationLookups()); + assertPlan(tenantConn, query).scanType("RANGE SCAN").table(tableName).keyRanges("['cd12']") + .serverSortedBy("[COL2]").clientSortAlgo("CLIENT MERGE SORT").numRegionLocationLookups(1); } try (Connection tenantConn = getTenantConnection("de12")) { @@ -528,14 +480,8 @@ public void testMultiTenantWithMetadataLookup() throws Exception { } assertEquals(25, c); - rs = tenantConn.createStatement().executeQuery("EXPLAIN " + query); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " ['de12']\n" - + " SERVER FILTER BY COL1 = 'col101'", queryPlan); - ExplainPlan plan = tenantConn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(1, planAttributes.getNumRegionLocationLookups()); + assertPlan(tenantConn, query).scanType("RANGE SCAN").table(tableName).keyRanges("['de12']") + .serverWhereFilter("SERVER FILTER BY COL1 = 'col101'").numRegionLocationLookups(1); } } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java index 4767ac34ba2..1c9a5a4bad2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java @@ -21,6 +21,7 @@ import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -34,7 +35,10 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; @@ -412,6 +416,63 @@ public static Estimate getByteRowEstimates(Connection conn, String sql, List= ?"; + List binds = Lists.newArrayList(); + binds.add(99); + try (Connection conn = DriverManager.getConnection(getUrl())) { + PhoenixPreparedStatement stmt = + conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class); + stmt.setInt(1, 99); + QueryPlan plan = stmt.optimizeQuery(); + Long planRows = plan.getEstimatedRowsToScan(); + Long planBytes = plan.getEstimatedBytesToScan(); + Long planTs = plan.getEstimateInfoTimestamp(); + assertEquals((Long) 10L, planRows); + assertEquals((Long) 634L, planBytes); + assertTrue(planTs > 0); + + // First-row EXPLAIN cells must equal the stats-driven values. + Estimate info = getByteRowEstimates(conn, sql, binds); + assertEquals(planRows, info.estimatedRows); + assertEquals(planBytes, info.estimatedBytes); + assertEquals(planTs, info.estimateInfoTs); + + // Top-of-plan attributes must equal the stats-driven values. + ExplainPlanAttributes attrs = plan.getExplainPlan().getPlanStepsAsAttributes(); + assertEquals(planRows, attrs.getEstimatedRows()); + assertEquals(planBytes, attrs.getEstimatedSizeInBytes()); + assertEquals(planTs, attrs.getEstimateInfoTs()); + } + } + + @Test + public void testEstimateCellsOnlyOnFirstRow() throws Exception { + // A query whose plan emits a SERVER FILTER BY step in addition to the scan line produces a + // multi row EXPLAIN result set with stats populated estimates at the top-of-plan. + String sql = "SELECT * FROM " + tableB + " WHERE k >= 99 AND c1.a > c2.b"; + try (Connection conn = DriverManager.getConnection(getUrl()); + PreparedStatement stmt = conn.prepareStatement("EXPLAIN " + sql); + ResultSet rs = stmt.executeQuery()) { + assertTrue("expected at least one EXPLAIN row", rs.next()); + assertNotNull(rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_ROWS_READ_COLUMN)); + assertNotNull(rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_BYTES_READ_COLUMN)); + assertNotNull(rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATE_INFO_TS_COLUMN)); + int extraRows = 0; + while (rs.next()) { + extraRows++; + rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_ROWS_READ_COLUMN); + assertTrue("EST_ROWS_READ must be NULL on row " + (extraRows + 1), rs.wasNull()); + rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_BYTES_READ_COLUMN); + assertTrue("EST_BYTES_READ must be NULL on row " + (extraRows + 1), rs.wasNull()); + rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATE_INFO_TS_COLUMN); + assertTrue("EST_INFO_TS must be NULL on row " + (extraRows + 1), rs.wasNull()); + } + assertTrue("expected multi-step plan", extraRows >= 1); + } + } + @Test public void testSettingUseStatsForParallelizationProperty() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl())) { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java index d84c3f6bb61..d30f79a2a78 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java @@ -17,9 +17,10 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlanWithRegions; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -42,24 +43,18 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionObserver; import org.apache.hadoop.hbase.util.Pair; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.index.BaseLocalIndexIT; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.experimental.categories.Category; +@Category(NeedsOwnMiniClusterTest.class) public class FlappingLocalIndexIT extends BaseLocalIndexIT { - private static final Logger LOGGER = LoggerFactory.getLogger(FlappingLocalIndexIT.class); - public FlappingLocalIndexIT(boolean isNamespaceMapped) { super(isNamespaceMapped); } @@ -173,25 +168,13 @@ public void testLocalIndexScan() throws Exception { String query = "SELECT * FROM " + tableName + " where v1 like 'a%'"; - String explainPlanOutput = QueryUtil - .getExplainPlan(conn1.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query)); - LOGGER.info("Explain plan output: {}", explainPlanOutput); - // MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN is set as 2 - assertTrue("Expected total " + numRegions + " regions", - explainPlanOutput.contains("...total size = " + numRegions)); - - ExplainPlan plan = conn1.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL " + numRegions + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexTableName + "(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,'a'] - [1,'b']", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(trimmedRegionLocations, explainPlanAttributes.getRegionLocations().size()); + // MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN is set as 2 so getRegionLocations() is trimmed. The + // full number of regions is reported via regionLocationsTotalSize. + assertPlanWithRegions(conn1, query).iteratorType("PARALLEL " + numRegions + "-WAY") + .scanType("RANGE SCAN").table(indexTableName + "(" + indexPhysicalTableName + ")") + .keyRanges("[1,'a'] - [1,'b']").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").regionLocationCount(trimmedRegionLocations) + .regionLocationsTotalSize(numRegions); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -209,17 +192,10 @@ public void testLocalIndexScan() throws Exception { assertFalse(rs.next()); query = "SELECT t_id, k1, k2,V1 FROM " + tableName + " where v1='a'"; - plan = conn1.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL " + numRegions + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexTableName + "(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,'a']", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn1, query).iteratorType("PARALLEL " + numRegions + "-WAY") + .scanType("RANGE SCAN").table(indexTableName + "(" + indexPhysicalTableName + ")") + .keyRanges("[1,'a']").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT"); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -233,18 +209,10 @@ public void testLocalIndexScan() throws Exception { assertFalse(rs.next()); query = "SELECT t_id, k1, k2,V1, k3 FROM " + tableName + " where v1<='z' order by k3"; - plan = conn1.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL " + numRegions + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexTableName + "(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,*] - [1,'z']", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals("[\"K3\"]", explainPlanAttributes.getServerSortedBy()); + assertPlan(conn1, query).iteratorType("PARALLEL " + numRegions + "-WAY") + .scanType("RANGE SCAN").table(indexTableName + "(" + indexPhysicalTableName + ")") + .keyRanges("[1,*] - [1,'z']").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").serverSortedBy("[\"K3\"]"); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -263,18 +231,10 @@ public void testLocalIndexScan() throws Exception { query = "SELECT t_id, k1, k2,v1 from " + tableName + " order by V1,t_id"; - plan = conn1.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL " + numRegions + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexTableName + "(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertNull(explainPlanAttributes.getServerSortedBy()); + assertPlan(conn1, query).iteratorType("PARALLEL " + numRegions + "-WAY") + .scanType("RANGE SCAN").table(indexTableName + "(" + indexPhysicalTableName + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .serverSortedBy(null); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java index 8d16b3a1c2f..8b3541e73c4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java @@ -19,6 +19,7 @@ import static java.util.Collections.singletonList; import static org.apache.phoenix.query.QueryServices.USE_BLOOMFILTER_FOR_MULTIKEY_POINTLOOKUP; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; @@ -46,14 +47,13 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.ExpressionCompiler; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.parse.ColumnParseNode; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; @@ -1024,20 +1024,16 @@ public void testPkDescOrderedTenantViewOnGlobalViewWithRightQueryPlan() throws E try (PreparedStatement preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID1, ID2) " + "IN (('005xx000001Sv6o', '000000000000500'))")) { - QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(PhoenixRuntime.getOptimizedQueryPlan(preparedStmt)) + .scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); } try (PreparedStatement preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID2, ID1) " + "IN (('000000000000500', '005xx000001Sv6o'))")) { - QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(PhoenixRuntime.getOptimizedQueryPlan(preparedStmt)) + .scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); } stmt.execute( @@ -1071,21 +1067,17 @@ public void testColumnDescOrderedTenantViewOnGlobalViewWithStringValue() throws try (PreparedStatement preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID1, ID2) " + "IN (('005xx000001Sv6o', '000000000000500')," + "('bar', '000000000000400')," + "('foo', '000000000000300'))")) { - QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(PhoenixRuntime.getOptimizedQueryPlan(preparedStmt)) + .scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); } try (PreparedStatement preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID2, ID1) IN " + "(('bar', '005xx000001Sv6o')," + "('foo', '005xx000001Sv6o'))")) { - QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(PhoenixRuntime.getOptimizedQueryPlan(preparedStmt)) + .scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); } stmt.execute("DELETE FROM " + tenantView + " WHERE (ID2, ID1) IN " @@ -1123,21 +1115,17 @@ public void testInListExpressionWithRightQueryPlanForTenantViewOnGlobalView() th viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID1, ID2) IN " + "(('005xx000001Sv6o', '000000000000500')," + "('005xx000001Sv6o', '000000000000400')," + "('005xx000001Sv6o', '000000000000300'))")) { - QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(PhoenixRuntime.getOptimizedQueryPlan(preparedStmt)) + .scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); } try (PreparedStatement preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID2, ID1) IN " + "(('000000000000400', '005xx000001Sv6o')," + "('000000000000300', '005xx000001Sv6o'))")) { - QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(PhoenixRuntime.getOptimizedQueryPlan(preparedStmt)) + .scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); } ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM " + tenantView + " WHERE (ID2, ID1) IN " + "(('000000000000400', '005xx000001Sv6o')," @@ -1178,19 +1166,15 @@ private void testFullPkListPlan(String tenantView) throws Exception { + "(('005xx000001Sv6o', '000000000000500')," + "('005xx000001Sv6o', '000000000000400'))"); QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); assertEquals(numberOfRowsToScan, queryPlan.getEstimatedRowsToScan()); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue( - explainPlanAttributes.getExplainScanType().startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(queryPlan).scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); viewConn.prepareStatement("DELETE FROM " + tenantView + " WHERE (ID1, ID2) IN " + "(('005xx000001Sv6o', '000000000000500')," + "('005xx000001Sv6o', '000000000000400'))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); assertEquals(numberOfRowsToScan, queryPlan.getEstimatedRowsToScan()); - plan = queryPlan.getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue( - explainPlanAttributes.getExplainScanType().startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(queryPlan).scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); } } @@ -1199,30 +1183,26 @@ private void testPartialPkListPlan(String tenantView) throws Exception { PreparedStatement preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID1) IN " + "(('005xx000001Sv6o')," + "('005xx000001Sv6o'))"); QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); viewConn.prepareStatement("DELETE FROM " + tenantView + " WHERE (ID1) IN " + "(('005xx000001Sv6o')," + "('005xx000001Sv6o'))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - assertTrue( - queryPlan.getExplainPlan().toString().contains("CLIENT PARALLEL 1-WAY RANGE SCAN OVER")); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID2) IN " + "(('000000000000500')," + "('000000000000400'))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - plan = queryPlan.getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); viewConn.prepareStatement("DELETE FROM " + tenantView + " WHERE (ID2) IN " + "(('000000000000500')," + "('000000000000400'))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - assertTrue( - queryPlan.getExplainPlan().toString().contains("CLIENT PARALLEL 1-WAY RANGE SCAN OVER")); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); } } @@ -1231,30 +1211,26 @@ private void testPartialPkPlusNonPkListPlan(String tenantView) throws Exception PreparedStatement preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID1, ID3) IN " + "(('005xx000001Sv6o', 1)," + "('005xx000001Sv6o', 2))"); QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); viewConn.prepareStatement("DELETE FROM " + tenantView + " WHERE (ID1, ID3) IN " + "(('005xx000001Sv6o', 1)," + "('005xx000001Sv6o', 2))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - assertTrue( - queryPlan.getExplainPlan().toString().contains("CLIENT PARALLEL 1-WAY RANGE SCAN OVER")); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); preparedStmt = viewConn.prepareStatement("SELECT * FROM " + tenantView + " WHERE (ID2, ID3) IN " + "(('000000000000500', 1)," + "('000000000000400', 2))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - plan = queryPlan.getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); viewConn.prepareStatement("DELETE FROM " + tenantView + " WHERE (ID2, ID3) IN " + "(('000000000000500', 1)," + "('000000000000400', 2))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - assertTrue( - queryPlan.getExplainPlan().toString().contains("CLIENT PARALLEL 1-WAY RANGE SCAN OVER")); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); } } @@ -1264,16 +1240,14 @@ private void testNonPkListPlan(String tenantView) throws Exception { PreparedStatement preparedStmt = viewConn.prepareStatement( "SELECT * FROM " + tenantView + " WHERE (ID3, ID4) IN " + "((1, 1)," + "(2, 2))"); QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); viewConn.prepareStatement( "DELETE FROM " + tenantView + " WHERE (ID3, ID4) IN " + "((1, 1)," + "(2, 2))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - assertTrue( - queryPlan.getExplainPlan().toString().contains("CLIENT PARALLEL 1-WAY RANGE SCAN OVER")); + assertPlan(queryPlan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); } } @@ -1775,28 +1749,22 @@ public void testBaseTableAndIndexTableHaveReversePKOrder() throws Exception { "SELECT * FROM " + view + " WHERE (ID1, ID2) IN " + "((1, 1)," + "(2, 2))"); QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); assertEquals(new Long(2), queryPlan.getEstimatedRowsToScan()); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(queryPlan).scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); preparedStmt = conn.prepareStatement("SELECT * FROM " + view + " WHERE (ID2, ID1) IN ((1, 1),(2, 2))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); assertEquals(new Long(2), queryPlan.getEstimatedRowsToScan()); - plan = queryPlan.getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(queryPlan).scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); preparedStmt = conn.prepareStatement("SELECT * FROM " + view + " WHERE (ID2, ID1) IN ((1, 1),(2, 2))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); assertEquals(new Long(2), queryPlan.getEstimatedRowsToScan()); - plan = queryPlan.getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(queryPlan).scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); ResultSet rs = stmt.executeQuery("SELECT ID1, ID2, ID5, ID4 FROM " + view + " WHERE (POWER(ID2, 2), ID1) IN " + "((4.0, 9)," + "(10, 12))"); @@ -1842,27 +1810,22 @@ public void testDeletionFromTenantViewAndViewIndex() throws Exception { PreparedStatement preparedStmt = conn.prepareStatement( "SELECT * FROM " + view + " WHERE (ID4, ID2) IN " + "((1, 1)," + "(2, 2))"); QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(queryPlan).scanType("RANGE SCAN").indexRule(OptimizerReasons.RULE_ONLY_CANDIDATE) + .indexRejectedCount(1); preparedStmt = conn.prepareStatement( "SELECT ID1,ID5 FROM " + view + " WHERE (ID1, ID2) IN " + "((1, 1),(2, 2))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); assertEquals(new Long(2), queryPlan.getEstimatedRowsToScan()); - plan = queryPlan.getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(queryPlan).scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); preparedStmt = conn.prepareStatement("SELECT * FROM " + view + " WHERE (ID2, ID1) IN ((1, 1),(2, 2))"); queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); assertEquals(new Long(2), queryPlan.getEstimatedRowsToScan()); - plan = queryPlan.getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue(explainPlanAttributes.getExplainScanType() - .startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(queryPlan).scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); ResultSet rs = stmt.executeQuery("SELECT ID1, ID2, ID5, ID4 FROM " + view + " WHERE (POWER(ID2, 2), ID1) IN " + "((4.0, 9)," + "(10, 12))"); @@ -1901,10 +1864,8 @@ public void testBaseTableAndIndexTableHaveRightScan() throws Exception { "SELECT VAL2 FROM " + fullTableName + " WHERE (ID2, ID1) IN ((1, 1),(2, 2))"); QueryPlan queryPlan = PhoenixRuntime.getOptimizedQueryPlan(preparedStmt); queryPlan.getTableRef().getTable().getType(); - ExplainPlan plan = queryPlan.getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertTrue( - explainPlanAttributes.getExplainScanType().startsWith(ExplainTable.POINT_LOOKUP_ON_STRING)); + assertPlan(queryPlan).scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING) + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); } } @@ -2246,12 +2207,13 @@ private void assertExpectedWithMaxInList(int tenantId, String testType, PDataTyp int lastBoundCol = 0; setBindVariables(stmt, lastBoundCol, numInLists, testPKTypes); QueryPlan plan = stmt.compileQuery(query.toString()); + // NOTE: compileQuery() does not run the QueryOptimizer, so the plan carries no + // OptimizerDecision (indexRule is null); intentionally no indexRule assertion here. if (expectSkipScan) { - assertTrue( - plan.getExplainPlan().toString().contains("CLIENT PARALLEL 1-WAY POINT LOOKUP ON")); + assertPlan(plan).iteratorType("PARALLEL 1-WAY") + .scanTypeStartsWith(ExplainTable.POINT_LOOKUP_ON_STRING); } else { - assertTrue( - plan.getExplainPlan().toString().contains("CLIENT PARALLEL 1-WAY RANGE SCAN OVER")); + assertPlan(plan).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN"); } ResultSet rs = stmt.executeQuery(query.toString()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java index e7ab684e8ba..3491b0f2a1b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -41,8 +42,8 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.util.EnvironmentEdge; import org.apache.phoenix.util.EnvironmentEdgeManager; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -136,11 +137,13 @@ public static synchronized Collection data() { public static void assertExplainPlan(Connection conn, boolean localIndex, String selectSql, String dataTableFullName, String indexTableFullName) throws SQLException { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - - IndexToolIT.assertExplainPlan(localIndex, actualExplainPlan, dataTableFullName, - indexTableFullName); + // Verify the query is served by a RANGE SCAN over the index table. For a local index the + // scanned name carries the data table in parentheses, e.g. IDX(DATA). + String expectedTable = localIndex + ? SchemaUtil.normalizeIdentifier(indexTableFullName) + "(" + + SchemaUtil.normalizeIdentifier(dataTableFullName) + ")" + : SchemaUtil.normalizeIdentifier(indexTableFullName); + assertPlan(conn, selectSql).scanType("RANGE SCAN").tableContains(expectedTable); } private class MyClock extends EnvironmentEdge { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java index d633c064e9e..90317a3cd0f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.checkIndexState; import static org.apache.phoenix.util.TestUtil.getRowCount; @@ -35,13 +36,10 @@ import java.util.Properties; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.mapreduce.index.IndexTool; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; @@ -177,15 +175,10 @@ public void testMutableIndexWithUpdates() throws Exception { String selectSql = String.format("SELECT ID FROM %s WHERE UPPER(NAME, 'en_US') ='UNAME2'", dataTableFullName); - ExplainPlan plan = conn.prepareStatement(selectSql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); // assert we are pulling from data table. - assertEquals(dataTableFullName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY UPPER(NAME, 'en_US') = 'UNAME2'", - explainPlanAttributes.getServerWhereFilter()); + assertPlan(conn, selectSql).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(dataTableFullName) + .serverWhereFilter("SERVER FILTER BY UPPER(NAME, 'en_US') = 'UNAME2'"); ResultSet rs = stmt1.executeQuery(selectSql); assertTrue(rs.next()); @@ -195,13 +188,10 @@ public void testMutableIndexWithUpdates() throws Exception { // run the index MR job. IndexToolIT.runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName); - plan = conn.prepareStatement(selectSql).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); // assert we are pulling from index table. String expectedTableName = localIndex ? indexTableFullName + "(" + dataTableFullName + ")" : indexTableFullName; - assertEquals(expectedTableName, explainPlanAttributes.getTableName()); + assertPlan(conn, selectSql).table(expectedTableName); rs = stmt.executeQuery(selectSql); assertTrue(rs.next()); @@ -259,13 +249,8 @@ public void testDeleteFromImmutable() throws Exception { // deleted String query = "SELECT pk3 from " + dataTableFullName + " ORDER BY pk3"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexTableFullName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(indexTableFullName).serverFirstKeyOnlyProjection(true); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForPartialBuildIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForPartialBuildIT.java index c4f3be5c383..c9f5c84bbed 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForPartialBuildIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForPartialBuildIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -39,10 +40,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.mapreduce.index.IndexTool; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; @@ -165,15 +163,10 @@ public void testSecondaryIndex() throws Exception { String selectSql = String.format("SELECT LPAD(UPPER(NAME),11,'x')||'_xyz',ID FROM %s", fullTableName); - ExplainPlan plan = conn.prepareStatement(selectSql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); // assert we are pulling from data table. - assertEquals(SchemaUtil - .getPhysicalHBaseTableName(schemaName, dataTableName, isNamespaceEnabled).toString(), - explainPlanAttributes.getTableName()); + assertPlan(conn, selectSql).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(SchemaUtil.getPhysicalHBaseTableName(schemaName, dataTableName, isNamespaceEnabled) + .toString()); rs = stmt1.executeQuery(selectSql); for (int i = 1; i <= 7; i++) { @@ -208,15 +201,9 @@ public void testSecondaryIndex() throws Exception { upsertRow(stmt1, 9000); conn.commit(); - plan = conn.prepareStatement(selectSql).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); // assert we are pulling from index table. - assertEquals( - SchemaUtil.getPhysicalHBaseTableName(schemaName, indxTable, isNamespaceEnabled).toString(), - explainPlanAttributes.getTableName()); + assertPlan(conn, selectSql).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table( + SchemaUtil.getPhysicalHBaseTableName(schemaName, indxTable, isNamespaceEnabled).toString()); rs = stmt.executeQuery(selectSql); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java index 23f083f0964..8f7c732c408 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java @@ -38,6 +38,8 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REPAIR_EXTRA_VERIFIED_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getExplainAttributes; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -82,11 +84,9 @@ import org.apache.hadoop.mapreduce.CounterGroup; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; -import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.index.IndexTestUtil; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.mapreduce.index.IndexTool; import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository; import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository; @@ -104,7 +104,6 @@ import org.apache.phoenix.util.IndexScrutiny; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -312,11 +311,8 @@ public void testSecondaryIndex() throws Exception { dataTableFullName); // assert we are pulling from data table. - ExplainPlan plan = conn.prepareStatement(selectSql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); + ExplainPlanAttributes explainPlanAttributes = assertPlan(conn, selectSql) + .iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").attributes(); assertEquals(dataTableFullName, explainPlanAttributes.getTableName().replaceAll(":", ".")); assertEquals( "SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'", @@ -345,10 +341,7 @@ public void testSecondaryIndex() throws Exception { QueryConstants.VERIFIED_BYTES); } // assert we are pulling from index table. - plan = conn.prepareStatement(selectSql).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + explainPlanAttributes = assertPlan(conn, selectSql).scanType("RANGE SCAN").attributes(); final String expectedTableName; if (localIndex) { expectedTableName = indexTableFullName + "(" + dataTableFullName + ")"; @@ -542,12 +535,8 @@ public void testIndexToolWithTenantId() throws Exception { runIndexTool(false, "", viewTenantName, indexNameTenant, tenantId, 0, new String[0]); String selectSql = String.format("SELECT ID FROM %s WHERE NAME='x'", viewTenantName); - ExplainPlan plan = connTenant.prepareStatement(selectSql) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(viewIndexTableName, explainPlanAttributes.getTableName()); + assertPlan(connTenant, selectSql).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(viewIndexTableName); ResultSet rs = connTenant.createStatement().executeQuery(selectSql); assertTrue(rs.next()); @@ -779,16 +768,18 @@ public void testCaseSensitiveNames() throws Exception { "SELECT ID FROM %s WHERE LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz' = 'xxUNAME2_xyz'", qDataTableName); - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - // assert we are pulling from data table. - assertEquals(String.format( - "CLIENT PARALLEL 1-WAY FULL SCAN OVER %s\n" - + " SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'", - dataTableNameForExplain), actualExplainPlan.replaceAll(":", ".")); - - rs = stmt1.executeQuery(selectSql); + ExplainPlanAttributes dataAttributes = getExplainAttributes(conn, selectSql); + assertPlan(dataAttributes).scanType("FULL SCAN").serverWhereFilter( + "SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'"); + // Table names are case-sensitive and (under namespace mapping) carry a ':' separator, so + // compare against the normalized ('.') form rather than via tableContains(). + String dataScanTable = dataAttributes.getTableName() == null + ? null + : dataAttributes.getTableName().replaceAll(":", "."); + assertEquals(dataTableNameForExplain, dataScanTable); + + ResultSet rs = stmt1.executeQuery(selectSql); assertTrue(rs.next()); assertEquals(2, rs.getInt(1)); assertFalse(rs.next()); @@ -803,12 +794,19 @@ public void testCaseSensitiveNames() throws Exception { conn.commit(); // assert we are pulling from index table. - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - actualExplainPlan = QueryUtil.getExplainPlan(rs); - // Because the explain plan doesn't include double-quotes around case-sensitive table names, - // we need to tell assertExplainPlan to not normalize our table names. - assertExplainPlan(localIndex, actualExplainPlan, dataTableNameForExplain, - indexTableNameForExplain, false); + ExplainPlanAttributes indexAttributes = getExplainAttributes(conn, selectSql); + assertPlan(indexAttributes).scanType("RANGE SCAN"); + // The explain plan doesn't include double-quotes around case-sensitive table names and (under + // namespace mapping) uses a ':' separator, so compare against the non-normalized ('.') form. + String indexScanTable = indexAttributes.getTableName() == null + ? null + : indexAttributes.getTableName().replaceAll(":", "."); + String expectedIndexTable = localIndex + ? indexTableNameForExplain + "(" + dataTableNameForExplain + ")" + : indexTableNameForExplain; + assertTrue( + "expected scanned table <" + indexScanTable + "> to use index <" + expectedIndexTable + ">", + indexScanTable != null && indexScanTable.contains(expectedIndexTable)); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); @@ -899,28 +897,40 @@ private void testIndexToDataVerificationHelper(boolean caseSensitive) throws Exc } } - public static void assertExplainPlan(boolean localIndex, String actualExplainPlan, - String dataTableFullName, String indexTableFullName) { - assertExplainPlan(localIndex, actualExplainPlan, dataTableFullName, indexTableFullName, true); + /** + * Verify the optimizer chose to scan {@code indexTableFullName}. For a local index the scanned + * "table" is reported as {@code IDX(DATA)} and the key range starts with {@code [1,}. + */ + public static void assertExplainPlan(Connection conn, boolean localIndex, String selectSql, + String dataTableFullName, String indexTableFullName) throws SQLException { + assertExplainPlan(conn, localIndex, selectSql, dataTableFullName, indexTableFullName, true); } - public static void assertExplainPlan(boolean localIndex, String actualExplainPlan, - String dataTableFullName, String indexTableFullName, boolean normalizeTableNames) { - String expectedExplainPlan; + public static void assertExplainPlan(Connection conn, boolean localIndex, String selectSql, + String dataTableFullName, String indexTableFullName, boolean normalizeTableNames) + throws SQLException { + String expectedTable; if (localIndex) { - expectedExplainPlan = String.format(" RANGE SCAN OVER %s [1,", - normalizeTableNames - ? SchemaUtil.normalizeIdentifier(indexTableFullName) + "(" - + SchemaUtil.normalizeIdentifier(dataTableFullName) + ")" - : indexTableFullName + "(" + dataTableFullName + ")"); + expectedTable = normalizeTableNames + ? SchemaUtil.normalizeIdentifier(indexTableFullName) + "(" + + SchemaUtil.normalizeIdentifier(dataTableFullName) + ")" + : indexTableFullName + "(" + dataTableFullName + ")"; } else { - expectedExplainPlan = String.format(" RANGE SCAN OVER %s", - normalizeTableNames - ? SchemaUtil.normalizeIdentifier(indexTableFullName) - : indexTableFullName); + expectedTable = normalizeTableNames + ? SchemaUtil.normalizeIdentifier(indexTableFullName) + : indexTableFullName; + } + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN"); + String actualTable = + attributes.getTableName() == null ? null : attributes.getTableName().replaceAll(":", "."); + assertTrue("expected scanned table <" + actualTable + "> to contain <" + expectedTable + ">", + actualTable != null && actualTable.contains(expectedTable)); + if (localIndex) { + String keyRanges = attributes.getKeyRanges(); + assertTrue("expected local-index key ranges <" + keyRanges + "> to start with '[1,'", + keyRanges != null && keyRanges.trim().startsWith("[1,")); } - assertTrue(actualExplainPlan + "\n expected to contain \n" + expectedExplainPlan, - actualExplainPlan.replaceAll(":", ".").contains(expectedExplainPlan)); } public static CounterGroup getMRJobCounters(IndexTool indexTool) throws IOException { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java index e500cb40210..584827d92e8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.analyzeTable; import static org.apache.phoenix.util.TestUtil.getAllSplits; @@ -32,9 +33,6 @@ import java.sql.Statement; import java.util.List; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.TestUtil; @@ -145,15 +143,8 @@ public void testQueryWithLimitAndStats() throws Exception { assertEquals(0, rs.getInt(1)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("SERIAL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals(1, explainPlanAttributes.getServerRowLimit().intValue()); - assertEquals(1, explainPlanAttributes.getClientRowLimit().intValue()); + assertPlan(conn, query).iteratorType("SERIAL 1-WAY").scanType("FULL SCAN").table(tableName) + .serverFirstKeyOnlyProjection(true).serverRowLimit(1L).clientRowLimit(1); } @Test diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java index e654fe0c19c..6c28084925c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -33,10 +34,7 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; @@ -143,33 +141,16 @@ public void testLocalIndexScanAfterRegionSplit() throws Exception { assertEquals(m, k1ColumnValue[m]); } - ExplainPlan plan = conn1.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL " + (4 + i) + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", - explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn1, query).iteratorType("PARALLEL " + (4 + i) + "-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT"); query = "SELECT t_id,k1,k3 FROM " + tableName; - plan = conn1.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals( - "PARALLEL " + ((strings[3 * i].compareTo("j") < 0) ? (4 + i) : (4 + i - 1)) + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "_2(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [2]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", - explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn1, query) + .iteratorType( + "PARALLEL " + ((strings[3 * i].compareTo("j") < 0) ? (4 + i) : (4 + i - 1)) + "-WAY") + .scanType("RANGE SCAN").table(fullIndexName + "_2(" + indexPhysicalTableName + ")") + .keyRanges("[2]").serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT"); rs = conn1.createStatement().executeQuery(query); Thread.sleep(1000); @@ -245,28 +226,14 @@ public void testLocalIndexScanAfterRegionsMerge() throws Exception { assertEquals(strings[j], rs.getString("V1")); } - ExplainPlan plan = conn1.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 3-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn1, query).iteratorType("PARALLEL 3-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT"); query = "SELECT t_id,k1,k3 FROM " + tableName; - plan = conn1.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 3-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "_2(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [2]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn1, query).iteratorType("PARALLEL 3-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "_2(" + indexPhysicalTableName + ")").keyRanges("[2]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT"); rs = conn1.createStatement().executeQuery(query); Thread.sleep(1000); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameExtendedIT.java index 4b8496d9cc0..9de4fd9f350 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameExtendedIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameExtendedIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; @@ -36,7 +37,6 @@ import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -216,13 +216,12 @@ public void testHint() throws Exception { populateTable(conn, tableName, 1, 2); String tableSelect = "SELECT V1,V2,V3 FROM " + tableName; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + tableSelect); - String plan = QueryUtil.getExplainPlan(rs); + String scannedTable = assertPlan(conn, tableSelect).attributes().getTableName(); // plan should use one of the indexes - assertEquals(true, plan.contains(indexName) || plan.contains(indexName2)); + assertEquals(true, scannedTable != null + && (scannedTable.contains(indexName) || scannedTable.contains(indexName2))); // Test hint for the other index - String hintedIndex = - QueryUtil.getExplainPlan(rs).contains(indexName) ? indexName2 : indexName; + String hintedIndex = scannedTable.contains(indexName) ? indexName2 : indexName; try (Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) { String snapshotName = new StringBuilder(hintedIndex).append("-Snapshot").toString(); admin.snapshot(snapshotName, TableName.valueOf(hintedIndex)); @@ -232,9 +231,8 @@ public void testHint() throws Exception { } String indexSelect = "SELECT /*+ INDEX(" + tableName + " " + hintedIndex + ")*/ V1,V2,V3 FROM " + tableName; - rs = conn.createStatement().executeQuery("EXPLAIN " + indexSelect); - assertEquals(true, QueryUtil.getExplainPlan(rs).contains(hintedIndex)); - rs = conn.createStatement().executeQuery(indexSelect); + assertPlan(conn, indexSelect).tableContains(hintedIndex); + ResultSet rs = conn.createStatement().executeQuery(indexSelect); assertEquals(true, rs.next()); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameIT.java index d8f9bec76b4..49c9c6b3b60 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameIT.java @@ -19,6 +19,7 @@ import static org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.INVALID_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.VALID_ROW_COUNT; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.MetaDataUtil.VIEW_INDEX_TABLE_PREFIX; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.*; @@ -42,7 +43,6 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.BeforeClass; import org.junit.Ignore; @@ -50,16 +50,12 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; @RunWith(Parameterized.class) @Category(NeedsOwnMiniClusterTest.class) public class LogicalTableNameIT extends LogicalTableNameBaseIT { - private static final Logger LOGGER = LoggerFactory.getLogger(LogicalTableNameIT.class); - protected boolean createChildAfterRename; private boolean immutable; private Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -213,8 +209,6 @@ public void testUpdatePhysicalIndexTableName_runScrutiny() throws Exception { String schemaName = "S_" + generateUniqueName(); String tableName = "TBL_" + generateUniqueName(); String indexName = "IDX_" + generateUniqueName(); - String fullTableName = SchemaUtil.getTableName(schemaName, tableName); - String fullIndexName = SchemaUtil.getTableName(schemaName, indexName); try (Connection conn = getConnection(props)) { try (Connection conn2 = getConnection(props)) { test_IndexTableChange(conn, conn2, schemaName, tableName, indexName, @@ -411,7 +405,6 @@ public void testWith2LevelViewsBaseTablePhysicalNameChange() throws Exception { String upsert = "UPSERT INTO " + fullLevel2ViewName + " (PK1, V1, VIEW_COL1, CHV2) VALUES (?,?,?,?)"; PreparedStatement upsertStmt = conn.prepareStatement(upsert); - ArrayList row = new ArrayList<>(); upsertStmt.setString(1, "PK10"); upsertStmt.setString(2, "V10"); upsertStmt.setString(3, "VIEW_COL1_10"); @@ -426,8 +419,7 @@ public void testWith2LevelViewsBaseTablePhysicalNameChange() throws Exception { String indexSelect = "SELECT chv2, V1, VIEW_COL1 FROM " + fullLevel2ViewName + " WHERE chv2='CHV210'"; - rs = conn2.createStatement().executeQuery("EXPLAIN " + indexSelect); - assertEquals(true, QueryUtil.getExplainPlan(rs).contains(VIEW_INDEX_TABLE_PREFIX)); + assertPlan(conn2, indexSelect).tableContains(VIEW_INDEX_TABLE_PREFIX); rs = conn2.createStatement().executeQuery(indexSelect); assertEquals(true, rs.next()); assertEquals(false, rs.next()); @@ -476,8 +468,7 @@ public void testHashJoin() throws Exception { } Object[] arr = HashJoinGlobalIndexIT.data().toArray(); String[] indexDDL = ((String[][]) arr[0])[0]; - String[] plans = ((String[][]) arr[0])[1]; - HashJoinGlobalIndexIT hjgit = new HashJoinGlobalIndexIT(indexDDL, plans); + HashJoinGlobalIndexIT hjgit = new HashJoinGlobalIndexIT(indexDDL); hjgit.createSchema(); hjgit.testInnerJoin(false); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java index 38a2c113db2..4df082dd5a8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.assertRawCellCount; import static org.apache.phoenix.util.TestUtil.assertRawRowCount; import static org.apache.phoenix.util.TestUtil.assertRowExistsAtSCN; @@ -52,7 +53,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.ManualEnvironmentEdge; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -685,8 +685,8 @@ private void createIndex(String dataTableName, String indexTableName, int indexV public static void assertExplainPlan(Connection conn, String selectSql, String dataTableFullName, String indexTableFullName) throws SQLException { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - IndexToolIT.assertExplainPlan(false, actualExplainPlan, dataTableFullName, indexTableFullName); + // Verify the query is served by a RANGE SCAN over the (global) index table. + assertPlan(conn, selectSql).scanType("RANGE SCAN") + .tableContains(SchemaUtil.normalizeIdentifier(indexTableFullName)); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java index 881a1198a34..fc1baa642e6 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.assertRawCellCount; import static org.apache.phoenix.util.TestUtil.assertRawRowCount; import static org.apache.phoenix.util.TestUtil.assertRowExistsAtSCN; @@ -28,7 +29,6 @@ import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Map; @@ -44,8 +44,8 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.ManualEnvironmentEdge; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.After; import org.junit.Assert; @@ -390,9 +390,9 @@ private void createIndex(String dataTableName, String indexTableName, int indexV public static void assertExplainPlan(Connection conn, String selectSql, String dataTableFullName, String indexTableFullName) throws SQLException { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - IndexToolIT.assertExplainPlan(false, actualExplainPlan, dataTableFullName, indexTableFullName); + // Verify the query is served by a RANGE SCAN over the (global) index table. + assertPlan(conn, selectSql).scanType("RANGE SCAN") + .tableContains(SchemaUtil.normalizeIdentifier(indexTableFullName)); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java index 6b9a15b024c..a5e36123977 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.analyzeTable; import static org.apache.phoenix.util.TestUtil.getAllSplits; @@ -36,7 +37,6 @@ import java.util.Properties; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Before; @@ -536,10 +536,7 @@ public void testBug4658() throws Exception { // Tests for FORWARD_SCAN hint String query = "SELECT /*+ FORWARD_SCAN */ * FROM " + tableName + " WHERE COL2 = 'AAA' ORDER BY COL1 DESC"; - try (ResultSet rs = stmt.executeQuery("EXPLAIN " + query)) { - String explainPlan = QueryUtil.getExplainPlan(rs); - assertFalse(explainPlan.contains("REVERSE")); - } + assertPlan(conn, query).clientSortedBy(null); try (ResultSet rs = stmt.executeQuery(query)) { assertTrue(rs.next()); assertEquals(rs.getString("COL1"), "222"); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OnDuplicateKey2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OnDuplicateKey2IT.java index b5a7b055f97..52d0e4ba4a3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OnDuplicateKey2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OnDuplicateKey2IT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -47,8 +48,6 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.VersionInfo; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; @@ -381,14 +380,12 @@ private void verifyIndexRow(Connection conn, String tableName, boolean deleted) } assertFalse(resultSet.next()); - ExplainPlan plan = - preparedStatement.unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(indexDDL.contains("index") + String expectedTable = indexDDL.contains("index") ? (indexDDL.contains("local index") ? tableName + "_IDX(" + tableName + ")" : tableName + "_IDX") - : tableName, explainPlanAttributes.getTableName()); + : tableName; + assertPlan(preparedStatement.unwrap(PhoenixPreparedStatement.class)).table(expectedTable); } private static void validateMultiRowDelete(String tableName, Connection conn, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OnDuplicateKeyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OnDuplicateKeyIT.java index 754933b5103..c004ccb40fd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OnDuplicateKeyIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OnDuplicateKeyIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -48,7 +49,7 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; +import org.apache.phoenix.util.SchemaUtil; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -561,10 +562,13 @@ public void run() { ResultSet rs; String selectSql = "SELECT * FROM " + tableName + " WHERE counter1 >= 0"; if (isIndexCreated) { - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - IndexToolIT.assertExplainPlan(this.indexDDL.contains("local"), actualExplainPlan, tableName, - tableName + "_IDX"); + // Verify the query is served by a RANGE SCAN over the index table. For a local index the + // scanned name carries the data table in parentheses, e.g. IDX(DATA). + boolean localIndex = this.indexDDL.contains("local"); + String index = SchemaUtil.normalizeIdentifier(tableName + "_IDX"); + String expectedTable = + localIndex ? index + "(" + SchemaUtil.normalizeIdentifier(tableName) + ")" : index; + assertPlan(conn, selectSql).scanType("RANGE SCAN").tableContains(expectedTable); } rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java index 8a1e55ad3ab..378498e6d29 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java @@ -17,20 +17,16 @@ */ package org.apache.phoenix.end2end; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.sql.Connection; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.QueryBuilder; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -72,12 +68,4 @@ protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder qu } return rs; } - - public static void validateQueryPlan(Connection conn, QueryBuilder queryBuilder, - String expectedPhoenixPlan, String expectedSparkPlan) throws SQLException { - if (StringUtils.isNotBlank(expectedPhoenixPlan)) { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + queryBuilder.build()); - assertEquals(expectedPhoenixPlan, QueryUtil.getExplainPlan(rs)); - } - } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java index 5de095496fe..e556f808d7c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.sql.Connection; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -37,7 +36,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -56,7 +54,6 @@ import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.QueryBuilder; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -253,14 +250,6 @@ protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder qu return rs; } - public static void validateQueryPlan(Connection conn, QueryBuilder queryBuilder, - String expectedPhoenixPlan, String expectedSparkPlan) throws SQLException { - if (StringUtils.isNotBlank(expectedPhoenixPlan)) { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + queryBuilder.build()); - assertEquals(expectedPhoenixPlan, QueryUtil.getExplainPlan(rs)); - } - } - protected static void assertResultSetWithRegionMoves(ResultSet rs, Object[][] rows, String tableName) throws Exception { for (int rowIndex = 0; rowIndex < rows.length; rowIndex++) { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ProjectArrayElemAfterHashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ProjectArrayElemAfterHashJoinIT.java index 40545af925d..18899bd3404 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ProjectArrayElemAfterHashJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ProjectArrayElemAfterHashJoinIT.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getExplainAttributes; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -28,8 +30,8 @@ import java.sql.ResultSet; import java.sql.Statement; import java.util.Properties; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -108,19 +110,12 @@ private String getQuery(String table, boolean fullArray, boolean hashJoin) { private void verifyExplain(Connection conn, String table, boolean fullArray, boolean hashJoin) throws Exception { - - String query = "EXPLAIN " + getQuery(table, fullArray, hashJoin); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery(query); - - try { - String plan = QueryUtil.getExplainPlan(rs); - assertTrue(plan != null); - assertTrue(fullArray || plan.contains("SERVER ARRAY ELEMENT PROJECTION")); - assertTrue(hashJoin == plan.contains("JOIN")); - } finally { - rs.close(); + String query = getQuery(table, fullArray, hashJoin); + ExplainPlanAttributes attributes = getExplainAttributes(conn, query); + if (!fullArray) { + assertPlan(attributes).serverParsedProjectionCount("ARRAY", 4); } + assertPlan(attributes).subPlanCount(hashJoin ? 1 : 0); } private void verifyResults(Connection conn, String table, boolean fullArray, boolean hashJoin) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java index 6504c92bcaf..5769da066db 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java @@ -117,7 +117,7 @@ public void testDebugLogs() throws Exception { // sleep for sometime to let query log committed Thread.sleep(delay); - try (ResultSet explainRS = conn.createStatement().executeQuery("Explain with regions " + query); + try (ResultSet explainRS = conn.createStatement().executeQuery("Explain " + query); ResultSet rs = conn.createStatement().executeQuery(logQuery)) { boolean foundQueryLog = false; @@ -127,6 +127,8 @@ public void testDebugLogs() throws Exception { assertEquals(rs.getString(BIND_PARAMETERS), null); assertEquals(rs.getString(USER), System.getProperty("user.name")); assertEquals(rs.getString(CLIENT_IP), InetAddress.getLocalHost().getHostAddress()); + // The EXPLAIN_PLAN column logged by the query logger must be equal to a freshly + // computed plan. assertEquals(rs.getString(EXPLAIN_PLAN), QueryUtil.getExplainPlan(explainRS)); assertEquals(rs.getString(GLOBAL_SCAN_DETAILS), context.getScan().toJSON()); assertEquals(rs.getLong(NO_OF_RESULTS_ITERATED), 10); @@ -299,8 +301,8 @@ private void testPreparedStatement(LogLevel loglevel) throws Exception { // sleep for sometime to let query log committed Thread.sleep(delay); - String explainQuery = - "EXPLAIN WITH REGIONS " + "SELECT * FROM " + tableName + " where V = 'value5'"; + // Compare against a plain EXPLAIN (no REGIONS) to match the logged plan. + String explainQuery = "EXPLAIN " + "SELECT * FROM " + tableName + " where V = 'value5'"; try (ResultSet explainRS = conn.createStatement().executeQuery(explainQuery); ResultSet rs = conn.createStatement().executeQuery(logQuery)) { boolean foundQueryLog = false; @@ -311,6 +313,8 @@ private void testPreparedStatement(LogLevel loglevel) throws Exception { loglevel == LogLevel.TRACE ? "value5" : null); assertEquals(rs.getString(USER), System.getProperty("user.name")); assertEquals(rs.getString(CLIENT_IP), InetAddress.getLocalHost().getHostAddress()); + // The EXPLAIN_PLAN column logged by the query logger must be equal to a freshly + // computed plan. assertEquals(rs.getString(EXPLAIN_PLAN), QueryUtil.getExplainPlan(explainRS)); assertEquals(rs.getString(GLOBAL_SCAN_DETAILS), context.getScan().toJSON()); assertEquals(rs.getLong(NO_OF_RESULTS_ITERATED), 1); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java index 2278f7341c8..cef4e03c8c8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithLimitIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.query.QueryServicesOptions.UNLIMITED_QUEUE_SIZE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -31,9 +32,6 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.RejectedExecutionException; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; @@ -91,15 +89,8 @@ public void testQueryWithLimitAndStats() throws Exception { assertEquals(0, rs.getInt(1)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("SERIAL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals(1, explainPlanAttributes.getServerRowLimit().intValue()); - assertEquals(1, explainPlanAttributes.getClientRowLimit().intValue()); + assertPlan(conn, query).iteratorType("SERIAL 1-WAY").scanType("FULL SCAN").table(tableName) + .serverFirstKeyOnlyProjection(true).serverRowLimit(1L).clientRowLimit(1); } finally { conn.close(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithOffsetIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithOffsetIT.java index 65ec34ff08c..232b9dc264a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithOffsetIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithOffsetIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -33,9 +34,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; import org.junit.Before; @@ -129,20 +127,13 @@ public void testOffsetSerialQueryExecutedOnServer() throws SQLException { initTableValues(conn); updateStatistics(conn); String query = "SELECT t_id from " + tableName + " offset " + offset; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); if (!isSalted) { - assertEquals("SERIAL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(offset, explainPlanAttributes.getServerOffset().intValue()); + assertPlan(conn, query).scanType("FULL SCAN").table(tableName) + .serverEmptyColumnOnlyProjection(true).iteratorType("SERIAL 1-WAY").serverOffset(offset); } else { - assertEquals("PARALLEL 10-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(offset, explainPlanAttributes.getClientOffset().intValue()); + assertPlan(conn, query).scanType("FULL SCAN").table(tableName) + .serverEmptyColumnOnlyProjection(true).iteratorType("PARALLEL 10-WAY") + .clientSortAlgo("CLIENT MERGE SORT").clientOffset(offset); } ResultSet rs = conn.createStatement().executeQuery(query); @@ -152,22 +143,16 @@ public void testOffsetSerialQueryExecutedOnServer() throws SQLException { assertEquals(STRINGS[offset + i - 1], rs.getString(1)); } query = "SELECT t_id from " + tableName + " ORDER BY v1 offset " + offset; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("[C2.V1]", explainPlanAttributes.getServerSortedBy()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(offset, explainPlanAttributes.getClientOffset().intValue()); if (!isSalted) { // When Parallel stats is actually disabled, it is PARALLEL 4-WAY // CLIENT PARALLEL 4-WAY FULL SCAN OVER T_N000001 // SERVER SORTED BY [C2.V1] CLIENT MERGE SORT CLIENT OFFSET 10 // When enabled, it is 5-WAY - assertEquals("PARALLEL 4-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + assertPlan(conn, query).scanType("FULL SCAN").table(tableName).serverSortedBy("[C2.V1]") + .clientSortAlgo("CLIENT MERGE SORT").clientOffset(offset).iteratorType("PARALLEL 4-WAY"); } else { - assertEquals("PARALLEL 10-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + assertPlan(conn, query).scanType("FULL SCAN").table(tableName).serverSortedBy("[C2.V1]") + .clientSortAlgo("CLIENT MERGE SORT").clientOffset(offset).iteratorType("PARALLEL 10-WAY"); } conn.close(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithTableSampleIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithTableSampleIT.java index 045e20aabfc..cebeb55eecb 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithTableSampleIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryWithTableSampleIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -26,12 +27,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.exception.PhoenixParserException; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Before; import org.junit.Test; @@ -56,7 +53,7 @@ public void testSingleQueryWrongSyntax() throws Exception { prepareTableWithValues(conn, 100); String query = "SELECT i1, i2 FROM " + tableName + " tablesample 15 "; - ResultSet rs = conn.createStatement().executeQuery(query); + conn.createStatement().executeQuery(query); } finally { conn.close(); } @@ -70,7 +67,7 @@ public void testSingleQueryWrongSamplingRate() throws Exception { prepareTableWithValues(conn, 100); String query = "SELECT i1, i2 FROM " + tableName + " tablesample (175) "; - ResultSet rs = conn.createStatement().executeQuery(query); + conn.createStatement().executeQuery(query); } finally { conn.close(); } @@ -214,14 +211,8 @@ public void testExplainSingleQuery() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl(), props)) { prepareTableWithValues(conn, 100); String query = "SELECT i1, i2 FROM " + tableName + " tablesample (45) "; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(0.45D, explainPlanAttributes.getSamplingRate(), 0D); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + assertPlan(conn, query).iteratorType("PARALLEL").scanType("FULL SCAN").samplingRate(0.45d) + .table(tableName).serverFirstKeyOnlyProjection(true); } } @@ -231,17 +222,14 @@ public void testExplainSingleQueryWithUnion() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), props); try { prepareTableWithValues(conn, 100); - String query = "EXPLAIN SELECT * FROM " + tableName - + " tablesample (100) where i1<2 union all SELECT * FROM " + tableName - + " tablesample (2) where i2<6000"; - ResultSet rs = conn.createStatement().executeQuery(query); - - assertEquals( - "UNION ALL OVER 2 QUERIES\n" + " CLIENT PARALLEL 1-WAY 1.0-SAMPLED RANGE SCAN OVER " - + tableName + " [*] - [2]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " CLIENT PARALLEL 1-WAY 0.02-SAMPLED FULL SCAN OVER " + tableName + "\n" - + " SERVER FILTER BY FIRST KEY ONLY AND I2 < 6000", - QueryUtil.getExplainPlan(rs)); + String query = + "SELECT * FROM " + tableName + " tablesample (100) where i1<2 union all SELECT * FROM " + + tableName + " tablesample (2) where i2<6000"; + assertPlan(conn, query).abstractExplainPlan("UNION ALL OVER 2 QUERIES").subPlanCount(2) + .subPlan(0).scanType("RANGE SCAN").table(tableName).keyRanges("[*] - [2]") + .samplingRate(1.0d).serverFirstKeyOnlyProjection(true).end().subPlan(1) + .scanType("FULL SCAN").table(tableName).samplingRate(0.02d) + .serverFirstKeyOnlyProjection(true).serverWhereFilter("SERVER FILTER BY I2 < 6000").end(); } finally { conn.close(); } @@ -253,17 +241,15 @@ public void testExplainSingleQueryWithJoins() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), props); try { prepareTableWithValues(conn, 100); - String query = "EXPLAIN SELECT count(*) FROM " + tableName + " as A tablesample (45), " + String query = "SELECT count(*) FROM " + tableName + " as A tablesample (45), " + joinedTableName + " as B tablesample (75) where A.i1=B.i1"; - System.out.println(query); - ResultSet rs = conn.createStatement().executeQuery(query); - assertEquals("CLIENT PARALLEL 1-WAY 0.45-SAMPLED FULL SCAN OVER " + tableName + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO SINGLE ROW\n" - + " PARALLEL INNER-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY 0.75-SAMPLED FULL SCAN OVER " + joinedTableName + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " DYNAMIC SERVER FILTER BY A.I1 IN (B.I1)", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).scanType("FULL SCAN").table(tableName).samplingRate(0.45d) + .serverFirstKeyOnlyProjection(true).serverAggregate("SERVER AGGREGATE INTO SINGLE ROW") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY A.I1 IN (B.I1)").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .scanType("FULL SCAN").table(joinedTableName).samplingRate(0.75d) + .serverFirstKeyOnlyProjection(true).end(); } finally { conn.close(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RTrimFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RTrimFunctionIT.java index b15d4d424e2..e0f04eeab6c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RTrimFunctionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RTrimFunctionIT.java @@ -17,8 +17,8 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; -import static org.junit.Assert.assertTrue; import java.sql.Connection; import java.sql.DriverManager; @@ -28,7 +28,6 @@ import java.util.Properties; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -66,8 +65,7 @@ private void testWithFixedLengthPK(SortOrder sortOrder, List expectedRes ResultSet rs = conn.createStatement().executeQuery(query); assertValueEqualsResultSet(rs, expectedResults); - rs = conn.createStatement().executeQuery("explain " + query); - assertTrue(QueryUtil.getExplainPlan(rs).contains("RANGE SCAN OVER " + tableName)); + assertPlan(conn, query).scanType("RANGE SCAN").table(tableName); conn.close(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java index 2fcf0c6a3ee..ebb42da8c62 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.ROW2; import static org.apache.phoenix.util.TestUtil.ROW3; import static org.apache.phoenix.util.TestUtil.ROW4; @@ -37,9 +38,7 @@ import java.sql.Statement; import java.util.Properties; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.util.PropertiesUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -81,15 +80,10 @@ public void testReverseRangeScan() throws Exception { assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY AND ENTITY_ID >= '00A323122312312'", - explainPlanAttributes.getServerWhereFilter()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").clientSortedBy("REVERSE") + .scanType("FULL SCAN").table(tableName).serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY ENTITY_ID >= '00A323122312312'") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); PreparedStatement statement = conn.prepareStatement("SELECT entity_id FROM " + tableName + " WHERE organization_id = ? AND entity_id >= ? ORDER BY organization_id DESC, entity_id DESC"); @@ -185,17 +179,10 @@ public void testReverseScanIndex() throws Exception { assertEquals(1, rs.getInt(1)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("SERIAL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexName, explainPlanAttributes.getTableName()); - assertEquals(" [not null]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals(1, explainPlanAttributes.getServerRowLimit().intValue()); - assertEquals(1, explainPlanAttributes.getClientRowLimit().intValue()); + assertPlan(conn, query).iteratorType("SERIAL 1-WAY").clientSortedBy("REVERSE") + .scanType("RANGE SCAN").table(indexName).keyRanges("[not null]") + .serverFirstKeyOnlyProjection(true).serverRowLimit(1L).clientRowLimit(1) + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java index a10cc8745d3..fe68966c842 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.ENTITYHISTID1; import static org.apache.phoenix.util.TestUtil.ENTITYHISTID3; import static org.apache.phoenix.util.TestUtil.ENTITYHISTID7; @@ -54,8 +55,6 @@ import java.util.Properties; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.util.DateUtil; @@ -1349,14 +1348,9 @@ public void testForceSkipScan() throws Exception { assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 4-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("SKIP SCAN ON 12 KEYS ", explainPlanAttributes.getExplainScanType()); - assertEquals(tempTableWithCompositePK, explainPlanAttributes.getTableName()); - assertEquals(" [X'00',2] - [X'03',4]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, query).iteratorType("PARALLEL 4-WAY").scanType("SKIP SCAN ON 12 KEYS") + .table(tempTableWithCompositePK).keyRanges("[X'00',2] - [X'03',4]") + .clientSortAlgo("CLIENT MERGE SORT"); } finally { conn.close(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorOffsetIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorOffsetIT.java index 1df20890569..590c793db99 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorOffsetIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorOffsetIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -41,7 +42,6 @@ import org.apache.phoenix.schema.RowValueConstructorOffsetNotCoercibleException; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -1079,16 +1079,8 @@ public void testIndexMultiColumnsIndexedFixedLengthNullLiteralsRVCOffset() throw @Test public void testOffsetExplain() throws SQLException { - String sql = - "EXPLAIN SELECT * FROM " + DATA_TABLE_NAME + " LIMIT 2 OFFSET (k1,k2,k3)=(2, 3, 2)"; - try (Statement statement = conn.createStatement(); ResultSet rs = statement.executeQuery(sql)) { - StringBuilder explainStringBuilder = new StringBuilder(); - while (rs.next()) { - String explain = rs.getString(1); - explainStringBuilder.append(explain); - } - assertTrue(explainStringBuilder.toString().contains("With RVC Offset")); - } + String sql = "SELECT * FROM " + DATA_TABLE_NAME + " LIMIT 2 OFFSET (k1,k2,k3)=(2, 3, 2)"; + assertPlan(conn, sql).hexStringRVCOffset("0x828383"); } @Test @@ -1271,12 +1263,8 @@ public void testRVCOffsetWithNotApplicableIndexHint() throws Exception { "SELECT /*+ INDEX(%s %s)*/ %s FROM %s " + "WHERE t_id = 'b' AND k1 = 2 AND k2 = 3 OFFSET (%s)=('a', 1, 2)", TABLE_NAME, INDEX_NAME, TABLE_ROW_KEY, TABLE_NAME, TABLE_ROW_KEY); - try (Statement statement = conn.createStatement()) { - ResultSet rs = statement.executeQuery("EXPLAIN " + sql); - String actualQueryPlan = QueryUtil.getExplainPlan(rs); - // As hinted plan is not applicable so use data plan which is point lookup - assertTrue(actualQueryPlan.contains("POINT LOOKUP ON 1 KEY OVER " + TABLE_NAME)); - } + // As hinted plan is not applicable so use data plan which is point lookup + assertPlan(conn, sql).scanType("POINT LOOKUP ON 1 KEY").table(TABLE_NAME); } @Test diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SequenceBulkAllocationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SequenceBulkAllocationIT.java index 85133397d4e..4080d5f30a0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SequenceBulkAllocationIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SequenceBulkAllocationIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; @@ -37,10 +38,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.exception.SQLExceptionCode; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.After; @@ -825,14 +823,8 @@ public void testExplainPlanForNextValuesFor() throws Exception { String query = "SELECT NEXT 1000 VALUES FOR " + sequenceName + " FROM " + tableName; // Assert output for Explain Plain result is as expected - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals(1, explainPlanAttributes.getClientSequenceCount().intValue()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(tableName) + .serverFirstKeyOnlyProjection(true).clientSequenceCount(1); } /** diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SequenceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SequenceIT.java index 3cc3a0510df..83f1fae5b95 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SequenceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SequenceIT.java @@ -20,6 +20,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE; import static org.apache.phoenix.query.QueryServicesTestImpl.DEFAULT_SEQUENCE_CACHE_SIZE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -35,12 +36,9 @@ import java.sql.Statement; import java.util.List; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesTestImpl; @@ -824,14 +822,8 @@ public void testExplainPlanValidatesSequences() throws Exception { Connection conn2 = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)); String query = "SELECT NEXT VALUE FOR " + sequenceName + " FROM " + tableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals(1, explainPlanAttributes.getClientSequenceCount().intValue()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(tableName) + .serverFirstKeyOnlyProjection(true).clientSequenceCount(1); ResultSet rs = conn.createStatement().executeQuery( "SELECT sequence_name, current_value FROM \"SYSTEM\".\"SEQUENCE\" WHERE sequence_name='" @@ -1484,20 +1476,14 @@ public void testNoFromClause() throws Exception { .execute("CREATE SEQUENCE " + secondSeqName + " START WITH 2 INCREMENT BY 3"); String query = "SELECT NEXT VALUE FOR " + seqName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(new Integer(1), explainPlanAttributes.getClientSequenceCount()); + assertPlan(conn, query).clientSequenceCount(1); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); query = "SELECT CURRENT VALUE FOR " + seqName; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(new Integer(1), explainPlanAttributes.getClientSequenceCount()); + assertPlan(conn, query).clientSequenceCount(1); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingIT.java index b86793badbe..175ea4cd73e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingIT.java @@ -17,11 +17,11 @@ */ package org.apache.phoenix.end2end; -import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.assertExplainPlan; import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.assertExplainPlanWithLimit; import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.commitWithException; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_PAGED_ROWS_COUNTER; import static org.apache.phoenix.query.QueryServices.USE_BLOOMFILTER_FOR_MULTIKEY_POINTLOOKUP; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -46,7 +46,6 @@ import org.apache.phoenix.coprocessor.PagingRegionScanner; import org.apache.phoenix.hbase.index.IndexRegionObserver; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.monitoring.MetricType; import org.apache.phoenix.query.QueryServices; @@ -55,7 +54,6 @@ import org.apache.phoenix.util.DateUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.Assert; import org.junit.BeforeClass; @@ -146,11 +144,9 @@ public void testScanWithLimit() throws Exception { conn.createStatement().execute(ddl); conn.commit(); + assertPlan(conn, "select * from " + tablename + " limit " + limit).tableContains(indexName); stmt = conn.prepareStatement("select * from " + tablename + " limit " + limit); try (ResultSet rs = stmt.executeQuery()) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexName)); int expectedRowCount = 0; int expectedId1 = 4; int expectedId2 = 0; @@ -434,10 +430,8 @@ public void testPagingWithUnverifiedIndexRows() throws Exception { } conn.commit(); String dql = String.format("SELECT count(*) from %s where k3 = 5", tablename); + assertPlan(conn, dql).tableContains(indexname); try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexname)); assertTrue(rs.next()); assertEquals(totalRows / 10, rs.getInt(1)); assertFalse(rs.next()); @@ -462,10 +456,8 @@ public void testPagingWithUnverifiedIndexRows() throws Exception { } finally { IndexRegionObserver.setFailDataTableUpdatesForTesting(false); } + assertPlan(conn, dql).tableContains(indexname); try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexname)); assertTrue(rs.next()); assertEquals(totalRows / 10, rs.getInt(1)); assertFalse(rs.next()); @@ -658,7 +650,7 @@ public void testUncoveredQuery() throws Exception { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0' GROUP BY val1"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertPlan(conn, selectSql).scanType("RANGE SCAN").tableContains(indexTableName); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(2, rs.getInt(1)); @@ -668,7 +660,7 @@ public void testUncoveredQuery() throws Exception { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0'"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertPlan(conn, selectSql).scanType("RANGE SCAN").tableContains(indexTableName); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(3, rs.getInt(1)); @@ -676,7 +668,7 @@ public void testUncoveredQuery() throws Exception { // Run an order by query where the uncovered index should be used selectSql = "SELECT val3 from " + dataTableName + " where val1 > '0' ORDER BY val1"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertPlan(conn, selectSql).scanType("RANGE SCAN").tableContains(indexTableName); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("abcd", rs.getString(1)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java index 24a5099134b..9f63f0c9881 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.assertResultSet; import static org.junit.Assert.assertEquals; @@ -30,7 +31,6 @@ import java.sql.Statement; import java.util.Properties; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -419,40 +419,31 @@ public void testBug2894() throws Exception { + " ) L\n" + " ON L.BUCKET = E.BUCKET AND L.TIMESTAMP = E.TIMESTAMP\n" + " ) C\n" + " GROUP BY C.BUCKET, C.TIMESTAMP ORDER BY C.BUCKET, C.TIMESTAMP"; - String p = i == 0 - ? "SORT-MERGE-JOIN (INNER) TABLES\n" - + " CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + eventCountTableName - + " [X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" - + " CLIENT MERGE SORT\n" + " CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" - + "AND (SKIP MERGE)\n" + " CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] - + " [X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']\n" - + " SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" - + " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, \"TIMESTAMP\", SRC_LOCATION, DST_LOCATION]\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, \"TIMESTAMP\", SRC_LOCATION, DST_LOCATION]\n" - + " CLIENT MERGE SORT\n" + " CLIENT SORTED BY [BUCKET, \"TIMESTAMP\"]\n" - + "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY [E.BUCKET, E.TIMESTAMP]" - : "SORT-MERGE-JOIN (INNER) TABLES\n" - + " CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + eventCountTableName - + " [X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]\n" - + " CLIENT MERGE SORT\n" + " CLIENT SORTED BY [BUCKET, TIMESTAMP]\n" - + "AND (SKIP MERGE)\n" + " CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] - + " [X'00','5SEC',1462993420000000001,'Tr/Bal'] - [X'01','5SEC',1462993520000000000,'Tr/Bal']\n" - + " SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" - + " SERVER DISTINCT PREFIX FILTER OVER [BUCKET, \"TIMESTAMP\", SRC_LOCATION, DST_LOCATION]\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, \"TIMESTAMP\", SRC_LOCATION, DST_LOCATION]\n" - + " CLIENT MERGE SORT\n" - + "CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY [E.BUCKET, E.TIMESTAMP]"; - - ResultSet rs = conn.createStatement().executeQuery("explain " + q); - assertEquals(p, QueryUtil.getExplainPlan(rs)); - - rs = conn.createStatement().executeQuery(q); + String lhsKeyRanges = + "[X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']"; + String rhsKeyRanges = i == 0 + ? lhsKeyRanges + : "[X'00','5SEC',1462993420000000001,'Tr/Bal'] - [X'01','5SEC',1462993520000000000,'Tr/Bal']"; + String rhsClientSortedBy = i == 0 ? "[BUCKET, \"TIMESTAMP\"]" : null; + + assertPlan(conn, q).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(true) + .clientAggregate("CLIENT AGGREGATE INTO ORDERED DISTINCT ROWS BY [E.BUCKET, E.TIMESTAMP]") + .lhs().iteratorType("PARALLEL").scanType("SKIP SCAN ON 2 RANGES") + .table(eventCountTableName).keyRanges(lhsKeyRanges).serverFirstKeyOnlyProjection(true) + .serverDistinctFilter("SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]") + .serverAggregate( + "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[BUCKET, TIMESTAMP]").end().rhs() + .iteratorType("PARALLEL").scanType("SKIP SCAN ON 2 RANGES").table(t[i]) + .keyRanges(rhsKeyRanges).serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY SRC_LOCATION = DST_LOCATION") + .serverDistinctFilter( + "SERVER DISTINCT PREFIX FILTER OVER [BUCKET, \"TIMESTAMP\", SRC_LOCATION, DST_LOCATION]") + .serverAggregate( + "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, \"TIMESTAMP\", SRC_LOCATION, DST_LOCATION]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy(rhsClientSortedBy).end(); + + ResultSet rs = conn.createStatement().executeQuery(q); assertTrue(rs.next()); assertEquals("5SEC", rs.getString(1)); assertEquals(1462993430000000000L, rs.getLong(2)); @@ -721,12 +712,8 @@ private static void verifyQueryPlanAndResultForBug4508(Connection conn, String p + "JOIN " + peopleTable + " ds ON ds.PERSON_ID = l.LOCALID"; for (String q : new String[] { query1, query2 }) { - ResultSet rs = conn.createStatement().executeQuery("explain " + q); - String plan = QueryUtil.getExplainPlan(rs); - assertFalse("Tables should not be sorted over their PKs:\n" + plan, - plan.contains("SERVER SORTED BY")); - - rs = conn.createStatement().executeQuery(q); + assertPlan(conn, q).lhs().serverSortedBy(null).end().rhs().serverSortedBy(null).end(); + ResultSet rs = conn.createStatement().executeQuery(q); assertTrue(rs.next()); assertEquals(2, rs.getInt(1)); assertFalse(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SpillableGroupByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SpillableGroupByIT.java index ab65682080f..3b14fe72efd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SpillableGroupByIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SpillableGroupByIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.createGroupByTestTable; import static org.junit.Assert.assertEquals; @@ -32,10 +33,7 @@ import java.sql.Statement; import java.util.Map; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.exception.SQLExceptionCode; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; @@ -191,13 +189,8 @@ public void testStatisticsAreNotWritten() throws SQLException { rs.close(); stmt.close(); String query = "SELECT * FROM " + tableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(new Integer(1), explainPlanAttributes.getSplitsChunk()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName, explainPlanAttributes.getTableName()); + assertPlan(conn, query).splitsChunk(1).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(tableName); conn.close(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubBinaryFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubBinaryFunctionIT.java index c2acef86dcf..2102e4ab056 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubBinaryFunctionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SubBinaryFunctionIT.java @@ -17,12 +17,13 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import org.apache.phoenix.util.QueryUtil; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -379,14 +380,11 @@ public void testExplainPlanWithSubBinaryFunctionInPK() throws Exception { count++; } Assert.assertEquals(2, count); - rs = conn.createStatement().executeQuery("EXPLAIN " + sql); - String plan = QueryUtil.getExplainPlan(rs); - Assert.assertTrue(plan.contains("RANGE SCAN OVER " + tableName + " [1,X'01'] - [1,X'02']")); + assertPlan(conn, sql).scanType("RANGE SCAN").table(tableName) + .keyRanges("[1,X'01'] - [1,X'02']"); sql = "SELECT * FROM " + tableName + " WHERE id = 1 AND SUBBINARY(VAR_BIN_COL, 2, 1) = X'01'"; - rs = conn.createStatement().executeQuery("EXPLAIN " + sql); - plan = QueryUtil.getExplainPlan(rs); - Assert.assertTrue(plan.contains("RANGE SCAN OVER " + tableName + " [1]")); + assertPlan(conn, sql).scanType("RANGE SCAN").table(tableName).keyRanges("[1]"); } private void upsertRow(PreparedStatement stmt, int id, byte[] b1, byte[] b2) throws SQLException { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableTTLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableTTLIT.java index dcefcb48669..5ba04afeac0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableTTLIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableTTLIT.java @@ -22,6 +22,7 @@ import static org.apache.phoenix.query.QueryConstants.CDC_PRE_IMAGE; import static org.apache.phoenix.query.QueryConstants.CDC_TTL_DELETE_EVENT_TYPE; import static org.apache.phoenix.query.QueryConstants.CDC_UPSERT_EVENT_TYPE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.*; import com.fasterxml.jackson.databind.ObjectMapper; @@ -45,7 +46,6 @@ import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.hbase.index.IndexRegionObserver; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PTable; @@ -575,9 +575,7 @@ public void testDeleteFamilyVersion() throws Exception { String expectedValue; String dql = "select val1, val2 from " + tableName + " where id = 'a1'"; try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertFalse(explainPlan.contains(indexName)); + assertPlan(conn, dql).table(tableName); assertTrue(rs.next()); indexColumnValue = rs.getString(1); expectedValue = rs.getString(2); @@ -609,9 +607,7 @@ public void testDeleteFamilyVersion() throws Exception { // do a read on the index which should trigger a read repair dql = "select val2 from " + tableName + " where val1 = '" + indexColumnValue + "'"; try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexName)); + assertPlan(conn, dql).tableContains(indexName); assertTrue(rs.next()); assertEquals(rs.getString(1), expectedValue); assertFalse(rs.next()); @@ -622,9 +618,7 @@ public void testDeleteFamilyVersion() throws Exception { TestUtil.dumpTable(conn, TableName.valueOf(indexName)); // run the same query again after compaction try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexName)); + assertPlan(conn, dql).tableContains(indexName); assertTrue(rs.next()); assertEquals(rs.getString(1), expectedValue); assertFalse(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java index 07ca7495754..338daf1e3cc 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java @@ -24,6 +24,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.schema.PTableType.SYSTEM; import static org.apache.phoenix.schema.PTableType.TABLE; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; @@ -61,7 +62,6 @@ import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.StringUtil; import org.apache.phoenix.util.TestUtil; @@ -694,19 +694,14 @@ public void testIndexHintWithTenantView() throws Exception { String sql = "SELECT /*+ INDEX(" + fullGrandChildViewName + " " + viewIndexName + ")*/ " + "val2, id2, val1, id3, id1 FROM " + fullGrandChildViewName + " WHERE id2 = 'a2' AND (id1 = 'a1' OR id1 = 'b1') AND id3 = 3"; - ResultSet rs = stmt.executeQuery("EXPLAIN " + sql); - String actualQueryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualQueryPlan - .contains("1-WAY POINT LOOKUP ON 2 KEYS OVER " + physicalViewIndexTableName)); - rs = stmt.executeQuery(sql); + assertPlan(conn, sql).scanType("POINT LOOKUP ON 2 KEYS") + .tableContains(physicalViewIndexTableName); + ResultSet rs = stmt.executeQuery(sql); assertTrue(rs.next()); assertFalse(rs.next()); sql = "SELECT val2, id2, val1, id3, id1 FROM " + fullGrandChildViewName + " WHERE id2 = 'a2' AND (id1 = 'a1' OR id1 = 'b1') AND id3 = 3"; - rs = stmt.executeQuery("EXPLAIN " + sql); - actualQueryPlan = QueryUtil.getExplainPlan(rs); - assertTrue( - actualQueryPlan.contains("1-WAY POINT LOOKUP ON 2 KEYS OVER " + fullDataTableName)); + assertPlan(conn, sql).scanType("POINT LOOKUP ON 2 KEYS").table(fullDataTableName); rs = stmt.executeQuery(sql); assertTrue(rs.next()); assertFalse(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java index dff28f344bd..938a9003c83 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.analyzeTable; import static org.apache.phoenix.util.TestUtil.getAllSplits; @@ -31,12 +32,10 @@ import java.sql.ResultSet; import java.util.List; import java.util.Properties; -import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -90,9 +89,7 @@ public void testPointLookupOnBaseTable() throws Exception { String dql = String.format("SELECT * FROM %s where org_id='%s' AND kp='%s' LIMIT 1", tableName, tenantId, kp); try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains("POINT LOOKUP ON 1 KEY")); + assertPlan(conn, dql).scanType("POINT LOOKUP ON 1 KEY"); assertTrue(rs.next()); assertEquals(tenantId, rs.getString(1)); assertEquals(kp, rs.getString(2)); @@ -100,9 +97,7 @@ public void testPointLookupOnBaseTable() throws Exception { dql = String.format("SELECT count(*) FROM %s where org_id='%s' AND kp='%s'", tableName, tenantId, kp); try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains("POINT LOOKUP ON 1 KEY")); + assertPlan(conn, dql).scanType("POINT LOOKUP ON 1 KEY"); assertTrue(rs.next()); assertEquals(nRows, rs.getInt(1)); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java index 35614d565d8..6bbb32b04c3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.MetaDataUtil.getViewIndexSequenceName; import static org.apache.phoenix.util.MetaDataUtil.getViewIndexSequenceSchemaName; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; @@ -36,12 +37,12 @@ import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.ColumnNotFoundException; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -206,11 +207,11 @@ private void createViewAndIndexesWithTenantId(String tableName, String viewName, .optimizeQuery().getExplainPlan(); ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + assertEquals("RANGE SCAN", explainPlanAttributes.getExplainScanType()); assertEquals( SchemaUtil.getPhysicalTableName(Bytes.toBytes(tableName), isNamespaceMapped).toString(), explainPlanAttributes.getTableName()); - assertEquals(" ['" + tenantId + "']", explainPlanAttributes.getKeyRanges()); + assertEquals("['" + tenantId + "']", explainPlanAttributes.getKeyRanges()); rs = conn.createStatement().executeQuery("select pk2,col1 from " + viewName + " where col1='f'"); @@ -223,14 +224,14 @@ private void createViewAndIndexesWithTenantId(String tableName, String viewName, .getExplainPlan(); explainPlanAttributes = plan.getPlanStepsAsAttributes(); assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + assertEquals("RANGE SCAN", explainPlanAttributes.getExplainScanType()); + assertTrue(explainPlanAttributes.isServerFirstKeyOnlyProjection()); if (localIndex) { assertEquals(fullIndexName + "(" + SchemaUtil.getPhysicalTableName(Bytes.toBytes(tableName), isNamespaceMapped).toString() + ")", explainPlanAttributes.getTableName()); assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(" [" + (1L + indexIdOffset) + ",'" + tenantId + "','f']", + assertEquals("[" + (1L + indexIdOffset) + ",'" + tenantId + "','f']", explainPlanAttributes.getKeyRanges()); } else { assertEquals( @@ -238,7 +239,7 @@ private void createViewAndIndexesWithTenantId(String tableName, String viewName, SchemaUtil.getPhysicalTableName(Bytes.toBytes(tableName), isNamespaceMapped).toBytes())), explainPlanAttributes.getTableName()); assertNull(explainPlanAttributes.getClientSortAlgo()); - assertEquals(" [" + (Short.MIN_VALUE + indexIdOffset) + ",'" + tenantId + "','f']", + assertEquals("[" + (Short.MIN_VALUE + indexIdOffset) + ",'" + tenantId + "','f']", explainPlanAttributes.getKeyRanges()); } @@ -358,27 +359,29 @@ public void testOverlappingDatesFilter() throws Exception { viewConn.createStatement() .execute("CREATE VIEW IF NOT EXISTS " + viewName + " AS SELECT * FROM " + tableName); - String query = "EXPLAIN SELECT PARENT_ID FROM " + viewName + " WHERE PARENT_TYPE='001' " + String expectedIndexName = SchemaUtil.getTableName(SCHEMA1, "IDX"); + String query1 = "SELECT PARENT_ID FROM " + viewName + " WHERE PARENT_TYPE='001' " + "AND (CREATED_DATE > to_date('2011-01-01') AND CREATED_DATE < to_date('2016-10-31'))" + "ORDER BY PARENT_TYPE,CREATED_DATE LIMIT 501"; - - ResultSet rs = viewConn.createStatement().executeQuery(query); - String exptectedIndexName = SchemaUtil.getTableName(SCHEMA1, "IDX"); - String expectedPlanFormat = "CLIENT SERIAL 1-WAY RANGE SCAN OVER " + exptectedIndexName - + " ['tenant1 ','001','%s 00:00:00.001'] - ['tenant1 ','001','%s 00:00:00.000']" - + "\n" + " SERVER FILTER BY FIRST KEY ONLY" + "\n" + " SERVER 501 ROW LIMIT" + "\n" - + "CLIENT 501 ROW LIMIT"; - assertEquals(String.format(expectedPlanFormat, "2011-01-01", "2016-10-31"), - QueryUtil.getExplainPlan(rs)); - - query = "EXPLAIN SELECT PARENT_ID FROM " + viewName + " WHERE PARENT_TYPE='001' " + assertPlan(viewConn, query1).iteratorType("SERIAL").scanType("RANGE SCAN") + .table(expectedIndexName) + .keyRanges("['tenant1 ','001','2011-01-01 00:00:00.001']" + + " - ['tenant1 ','001','2016-10-31 00:00:00.000']") + .serverFirstKeyOnlyProjection(true).serverRowLimit(501L).clientRowLimit(501) + .clientSteps("CLIENT 501 ROW LIMIT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); + + String query2 = "SELECT PARENT_ID FROM " + viewName + " WHERE PARENT_TYPE='001' " + " AND (CREATED_DATE >= to_date('2011-01-01') AND CREATED_DATE <= to_date('2016-01-01'))" + " AND (CREATED_DATE > to_date('2012-10-21') AND CREATED_DATE < to_date('2016-10-31')) " + "ORDER BY PARENT_TYPE,CREATED_DATE LIMIT 501"; - - rs = viewConn.createStatement().executeQuery(query); - assertEquals(String.format(expectedPlanFormat, "2012-10-21", "2016-01-01"), - QueryUtil.getExplainPlan(rs)); + assertPlan(viewConn, query2).iteratorType("SERIAL").scanType("RANGE SCAN") + .table(expectedIndexName) + .keyRanges("['tenant1 ','001','2012-10-21 00:00:00.001']" + + " - ['tenant1 ','001','2016-01-01 00:00:00.000']") + .serverFirstKeyOnlyProjection(true).serverRowLimit(501L).clientRowLimit(501) + .clientSteps("CLIENT 501 ROW LIMIT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithDisabledIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithDisabledIndexIT.java index a1372be5496..33deda98211 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithDisabledIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithDisabledIndexIT.java @@ -19,6 +19,7 @@ import static org.apache.phoenix.query.QueryServices.DISABLE_VIEW_SUBTREE_VALIDATION; import static org.apache.phoenix.query.QueryServices.INDEX_USE_SERVER_METADATA_ATTRIB; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import com.google.protobuf.RpcCallback; import com.google.protobuf.RpcController; @@ -31,15 +32,12 @@ import java.util.Properties; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.coprocessor.MetaDataEndpointImpl; import org.apache.phoenix.coprocessor.generated.MetaDataProtos; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.exception.PhoenixIOException; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.protobuf.ProtobufUtil; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; @@ -172,10 +170,7 @@ public void testUcfWithNoGetTableCalls() throws Throwable { Assert.assertTrue(rs.next()); Assert.assertFalse(rs.next()); - ExplainPlan explainPlan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - Assert.assertEquals(indexName, explainPlanAttributes.getTableName()); + assertPlan(conn, query).table(indexName); TestUtil.removeCoprocessor(conn, "SYSTEM.CATALOG", TestMetaDataEndpointImpl.class); TestUtil.addCoprocessor(conn, "SYSTEM.CATALOG", MetaDataEndpointImpl.class); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UnionAllIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UnionAllIT.java index 208e18f88e2..293208dcf85 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UnionAllIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UnionAllIT.java @@ -17,10 +17,10 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -30,7 +30,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; +import java.util.List; import java.util.Properties; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; @@ -38,7 +38,6 @@ import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -626,52 +625,41 @@ public void testExplainUnionAll() throws Exception { ExplainPlan plan = conn.prepareStatement(ddl).unwrap(PhoenixPreparedStatement.class) .optimizeQuery().getExplainPlan(); ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); + // The union composes each branch recursively into subPlans. The root carries only the union + // level + // client side, and each branch's scan attributes live on its own subPlan entry. assertEquals("UNION ALL OVER 2 QUERIES", explainPlanAttributes.getAbstractExplainPlan()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName1, explainPlanAttributes.getTableName()); - assertEquals("[COL1]", explainPlanAttributes.getServerSortedBy()); - assertEquals(1L, explainPlanAttributes.getServerRowLimit().longValue()); - assertEquals(1, explainPlanAttributes.getClientRowLimit().intValue()); assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - ExplainPlanAttributes rhsPlanAttributes = explainPlanAttributes.getRhsJoinQueryExplainPlan(); + assertEquals(1, explainPlanAttributes.getClientRowLimit().intValue()); + List subPlans = explainPlanAttributes.getSubPlans(); + assertEquals(2, subPlans.size()); + ExplainPlanAttributes lhsPlanAttributes = subPlans.get(0); + assertEquals("PARALLEL 1-WAY", lhsPlanAttributes.getIteratorTypeAndScanSize()); + assertEquals("FULL SCAN", lhsPlanAttributes.getExplainScanType()); + assertEquals(tableName1, lhsPlanAttributes.getTableName()); + assertEquals("[COL1]", lhsPlanAttributes.getServerSortedBy()); + assertEquals(1L, lhsPlanAttributes.getServerRowLimit().longValue()); + assertEquals(1, lhsPlanAttributes.getClientRowLimit().intValue()); + assertEquals("CLIENT MERGE SORT", lhsPlanAttributes.getClientSortAlgo()); + ExplainPlanAttributes rhsPlanAttributes = subPlans.get(1); assertEquals("PARALLEL 1-WAY", rhsPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", rhsPlanAttributes.getExplainScanType()); + assertEquals("FULL SCAN", rhsPlanAttributes.getExplainScanType()); assertEquals(tableName2, rhsPlanAttributes.getTableName()); assertEquals("[COL1]", rhsPlanAttributes.getServerSortedBy()); assertEquals(1L, rhsPlanAttributes.getServerRowLimit().longValue()); assertEquals(1, rhsPlanAttributes.getClientRowLimit().intValue()); assertEquals("CLIENT MERGE SORT", rhsPlanAttributes.getClientSortAlgo()); - String limitPlan = "UNION ALL OVER 2 QUERIES\n" + " CLIENT SERIAL 1-WAY FULL SCAN OVER " - + tableName1 + "\n" + " SERVER 2 ROW LIMIT\n" + " CLIENT 2 ROW LIMIT\n" - + " CLIENT SERIAL 1-WAY FULL SCAN OVER " + tableName2 + "\n" - + " SERVER 2 ROW LIMIT\n" + " CLIENT 2 ROW LIMIT\n" + "CLIENT 2 ROW LIMIT"; - + // Each branch emits a SERIAL 1-WAY FULL SCAN with SERVER 2 ROW LIMIT and an inner CLIENT 2 + // ROW LIMIT. The union root carries the single outer CLIENT 2 ROW LIMIT. ddl = "select a_string, col1 from " + tableName1 + " union all select a_string, col1 from " + tableName2 + " limit 2"; - plan = conn.prepareStatement(ddl).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("UNION ALL OVER 2 QUERIES", explainPlanAttributes.getAbstractExplainPlan()); - assertEquals("SERIAL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName1, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getServerSortedBy()); - assertEquals(2L, explainPlanAttributes.getServerRowLimit().longValue()); - assertEquals(2, explainPlanAttributes.getClientRowLimit().intValue()); - rhsPlanAttributes = explainPlanAttributes.getRhsJoinQueryExplainPlan(); - assertEquals("SERIAL 1-WAY", rhsPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", rhsPlanAttributes.getExplainScanType()); - assertEquals(tableName2, rhsPlanAttributes.getTableName()); - assertNull(rhsPlanAttributes.getServerSortedBy()); - assertEquals(2L, rhsPlanAttributes.getServerRowLimit().longValue()); - assertEquals(2, rhsPlanAttributes.getClientRowLimit().intValue()); - - Statement stmt = conn.createStatement(); - stmt.setMaxRows(2); - ResultSet rs = stmt.executeQuery("explain " + ddl); - assertEquals(limitPlan, QueryUtil.getExplainPlan(rs)); + assertPlan(conn, ddl).abstractExplainPlan("UNION ALL OVER 2 QUERIES").clientRowLimit(2) + .clientSteps("CLIENT 2 ROW LIMIT").subPlanCount(2).subPlan(0).iteratorType("SERIAL") + .scanType("FULL SCAN").table(tableName1).serverSortedBy(null).serverRowLimit(2L) + .clientRowLimit(2).clientSteps("CLIENT 2 ROW LIMIT").end().subPlan(1).iteratorType("SERIAL") + .scanType("FULL SCAN").table(tableName2).serverSortedBy(null).serverRowLimit(2L) + .clientRowLimit(2).clientSteps("CLIENT 2 ROW LIMIT").end(); ddl = "select a_string, col1 from " + tableName1 + " union all select a_string, col1 from " + tableName2; @@ -679,12 +667,15 @@ public void testExplainUnionAll() throws Exception { .getExplainPlan(); explainPlanAttributes = plan.getPlanStepsAsAttributes(); assertEquals("UNION ALL OVER 2 QUERIES", explainPlanAttributes.getAbstractExplainPlan()); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(tableName1, explainPlanAttributes.getTableName()); - rhsPlanAttributes = explainPlanAttributes.getRhsJoinQueryExplainPlan(); + subPlans = explainPlanAttributes.getSubPlans(); + assertEquals(2, subPlans.size()); + lhsPlanAttributes = subPlans.get(0); + assertEquals("PARALLEL 1-WAY", lhsPlanAttributes.getIteratorTypeAndScanSize()); + assertEquals("FULL SCAN", lhsPlanAttributes.getExplainScanType()); + assertEquals(tableName1, lhsPlanAttributes.getTableName()); + rhsPlanAttributes = subPlans.get(1); assertEquals("PARALLEL 1-WAY", rhsPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", rhsPlanAttributes.getExplainScanType()); + assertEquals("FULL SCAN", rhsPlanAttributes.getExplainScanType()); assertEquals(tableName2, rhsPlanAttributes.getTableName()); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeNamespaceIT.java index 31b3f7524c8..0cd7923734a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeNamespaceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeNamespaceIT.java @@ -323,7 +323,19 @@ public boolean prune(PFunction function) { public void assertTableUsed(Connection conn, String phoenixTableName, String hbaseTableName) throws SQLException { ResultSet rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + phoenixTableName); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(hbaseTableName)); + // The EXPLAIN output may prepend top-of-plan disclosure lines (e.g. TENANT '...') before the + // SCAN line that carries the physical HBase table name, so scan every row of the plan rather + // than only the first. + boolean found = false; + while (rs.next()) { + String step = rs.getString(1); + if (step != null && step.contains(hbaseTableName)) { + found = true; + break; + } + } + assertTrue( + "EXPLAIN of " + phoenixTableName + " did not reference physical table " + hbaseTableName, + found); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java index 1bf090cdae1..ebeb76ca3e9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java @@ -19,6 +19,7 @@ import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_OPEN_PHOENIX_CONNECTIONS; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertMutationPlan; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.apache.phoenix.util.PhoenixRuntime.UPSERT_BATCH_SIZE_ATTRIB; import static org.apache.phoenix.util.TestUtil.A_VALUE; @@ -51,6 +52,7 @@ import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.monitoring.GlobalMetric; @@ -60,7 +62,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.TestUtil; import org.junit.After; @@ -182,12 +183,10 @@ private void testUpsertSelect(boolean createIndex, boolean saltTable) throws Exc + "SELECT substr(entity_id, 4), substr(entity_id, 1, 3), organization_id, " + "a_string FROM " + aTable + " WHERE ?=a_string"; if (createIndex) { // Confirm index is used - try (PreparedStatement upsertStmt = conn.prepareStatement("EXPLAIN " + upsert)) { + try (PhoenixPreparedStatement upsertStmt = + conn.prepareStatement(upsert).unwrap(PhoenixPreparedStatement.class)) { upsertStmt.setString(1, tenantId); - ResultSet ers = upsertStmt.executeQuery(); - assertTrue(ers.next()); - String explainPlan = QueryUtil.getExplainPlan(ers); - assertTrue(explainPlan.contains(" SCAN OVER " + indexName)); + assertMutationPlan(upsertStmt).tableContains(indexName); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java index c2ffbc30a57..3098be41994 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertMutationPlan; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.apache.phoenix.util.PhoenixRuntime.UPSERT_BATCH_SIZE_ATTRIB; import static org.apache.phoenix.util.TestUtil.A_VALUE; @@ -52,6 +53,7 @@ import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.monitoring.GlobalMetric; @@ -61,7 +63,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.TestUtil; import org.junit.After; @@ -190,12 +191,10 @@ private void testUpsertSelect(boolean createIndex, boolean saltTable) throws Exc + "SELECT substr(entity_id, 4), substr(entity_id, 1, 3), organization_id, " + "a_string FROM " + aTable + " WHERE ?=a_string"; if (createIndex) { // Confirm index is used - try (PreparedStatement upsertStmt = conn.prepareStatement("EXPLAIN " + upsert)) { + try (PhoenixPreparedStatement upsertStmt = + conn.prepareStatement(upsert).unwrap(PhoenixPreparedStatement.class)) { upsertStmt.setString(1, tenantId); - ResultSet ers = upsertStmt.executeQuery(); - assertTrue(ers.next()); - String explainPlan = QueryUtil.getExplainPlan(ers); - assertTrue(explainPlan.contains(" SCAN OVER " + indexName)); + assertMutationPlan(upsertStmt).tableContains(indexName); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java index 4f174bb2b2f..0af1cb18fe1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java @@ -20,6 +20,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE; import static org.apache.phoenix.query.QueryServices.DYNAMIC_JARS_DIR_KEY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK; @@ -55,10 +56,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.expression.function.UDFExpression; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixTestDriver; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.FunctionAlreadyExistsException; @@ -842,13 +840,8 @@ public void testFunctionalIndexesWithUDFFunction() throws Exception { stmt.execute("create index idx on t5(myreverse5(lastname_reverse))"); String query = "select myreverse5(lastname_reverse) from t5"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("IDX", explainPlanAttributes.getTableName()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table("IDX") + .serverFirstKeyOnlyProjection(true); ResultSet rs = stmt.executeQuery(query); assertTrue(rs.next()); @@ -858,15 +851,9 @@ public void testFunctionalIndexesWithUDFFunction() throws Exception { query = "select k,k1,myreverse5(lastname_reverse) from t5 where myreverse5(lastname_reverse)='kcoj'"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("IDX2(T5)", explainPlanAttributes.getTableName()); - assertEquals(" [1,'kcoj']", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table("IDX2(T5)") + .keyRanges("[1,'kcoj']").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT"); rs = stmt.executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VarBinaryEncoded2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VarBinaryEncoded2IT.java index 6b05b7dd0a5..205801164a9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VarBinaryEncoded2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VarBinaryEncoded2IT.java @@ -20,6 +20,7 @@ import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_RETRY_PAUSE_MS; import static org.apache.phoenix.hbase.index.IndexCDCConsumer.INDEX_CDC_CONSUMER_TIMESTAMP_BUFFER_MS; import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_MUTATION_SERIALIZE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import java.sql.Connection; @@ -33,8 +34,6 @@ import java.util.Map; import java.util.Properties; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryServices; @@ -1275,12 +1274,8 @@ public void testVarBinaryPkSchema4() throws Exception { private static void assertIndexUsed(PreparedStatement preparedStatement, String indexName, String scanType) throws SQLException { - ExplainPlan plan = - preparedStatement.unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = plan.getPlanStepsAsAttributes(); - - Assert.assertEquals(indexName, planAttributes.getTableName()); - Assert.assertEquals(scanType, planAttributes.getExplainScanType()); + assertPlan(preparedStatement.unwrap(PhoenixPreparedStatement.class)).table(indexName) + .scanType(scanType); } private static void upsertRow(PreparedStatement preparedStatement, byte[] b10, byte[] b20, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java index c58946caae5..b08ec1158a8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java @@ -18,13 +18,13 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.coprocessor.PhoenixMetaDataCoprocessorHost.PHOENIX_META_DATA_COPROCESSOR_CONF_KEY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.thirdparty.com.google.common.collect.Lists.newArrayListWithExpectedSize; import static org.apache.phoenix.util.TestUtil.analyzeTable; import static org.apache.phoenix.util.TestUtil.getAllSplits; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -49,12 +49,9 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.util.Pair; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; @@ -523,26 +520,13 @@ private void testViewUsesTableIndex(boolean localIndex) throws Exception { assertEquals(100, rs.getInt(1)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("SKIP SCAN ON 4 KEYS ", explainPlanAttributes.getExplainScanType()); - assertEquals("SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')", - explainPlanAttributes.getServerWhereFilter()); - - // Assert that in either case (local & global) that index from - // physical table used for query on view. - if (localIndex) { - assertEquals(fullIndexName1 + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,1,100] - [1,2,109]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - } else { - assertEquals(fullIndexName1, explainPlanAttributes.getTableName()); - assertEquals(" [1,100] - [2,109]", explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); - } + String expectedTable = + localIndex ? fullIndexName1 + "(" + fullTableName + ")" : fullIndexName1; + String expectedKeyRanges = localIndex ? "[1,1,100] - [1,2,109]" : "[1,100] - [2,109]"; + String expectedClientSortAlgo = localIndex ? "CLIENT MERGE SORT" : null; + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("SKIP SCAN ON 4 KEYS") + .serverWhereFilter("SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')") + .table(expectedTable).keyRanges(expectedKeyRanges).clientSortAlgo(expectedClientSortAlgo); } } @@ -979,36 +963,22 @@ public static Pair testUpdatableViewIndex(String fullTableName, In assertEquals("bar", rs.getString(4)); assertFalse(rs.next()); - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - if (localIndex) { - assertEquals("PARALLEL " + (saltBuckets == null ? 1 : saltBuckets) + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(viewIndexFullName1 + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,51]", explainPlanAttributes.getKeyRanges()); - assertTrue( - explainPlanAttributes.getServerWhereFilter().equals("SERVER FILTER BY FIRST KEY ONLY") - || explainPlanAttributes.getServerWhereFilter() - .equals("SERVER FILTER BY EMPTY COLUMN ONLY")); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, query).scanType("RANGE SCAN") + .iteratorType("PARALLEL " + (saltBuckets == null ? 1 : saltBuckets) + "-WAY") + .table(viewIndexFullName1 + "(" + fullTableName + ")").keyRanges("[1,51]") + .serverProjectionFilterAnyOf().clientSortAlgo("CLIENT MERGE SORT"); + } else if (saltBuckets == null) { + assertPlan(conn, query).scanType("RANGE SCAN").iteratorType("PARALLEL 1-WAY") + .table(viewIndexPhysicalName).keyRanges("[" + Short.MIN_VALUE + ",51]") + .clientSortAlgo(null); } else { - assertEquals(viewIndexPhysicalName, explainPlanAttributes.getTableName()); - if (saltBuckets == null) { - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(" [" + Short.MIN_VALUE + ",51]", explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); - } else { - assertEquals("PARALLEL " + saltBuckets + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(" [X'00'," + Short.MIN_VALUE + ",51] - [" + assertPlan(conn, query).scanType("RANGE SCAN") + .iteratorType("PARALLEL " + saltBuckets + "-WAY").table(viewIndexPhysicalName) + .keyRanges("[X'00'," + Short.MIN_VALUE + ",51] - [" + PVarbinary.INSTANCE.toStringLiteral(new byte[] { (byte) (saltBuckets - 1) }) + "," - + Short.MIN_VALUE + ",51]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - } + + Short.MIN_VALUE + ",51]") + .clientSortAlgo("CLIENT MERGE SORT"); } String viewIndexName2 = "I_" + generateUniqueName(); @@ -1038,37 +1008,24 @@ public static Pair testUpdatableViewIndex(String fullTableName, In assertFalse(rs.next()); String physicalTableName; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertTrue( - explainPlanAttributes.getServerWhereFilter().equals("SERVER FILTER BY FIRST KEY ONLY") - || explainPlanAttributes.getServerWhereFilter() - .equals("SERVER FILTER BY EMPTY COLUMN ONLY")); if (localIndex) { physicalTableName = fullTableName; - assertEquals("PARALLEL " + (saltBuckets == null ? 1 : saltBuckets) + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(viewIndexFullName2 + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [" + (2) + ",'foo']", explainPlanAttributes.getKeyRanges()); + assertPlan(conn, query).scanType("RANGE SCAN").serverProjectionFilterAnyOf() + .iteratorType("PARALLEL " + (saltBuckets == null ? 1 : saltBuckets) + "-WAY") + .table(viewIndexFullName2 + "(" + fullTableName + ")").keyRanges("[" + (2) + ",'foo']"); + } else if (saltBuckets == null) { + physicalTableName = viewIndexPhysicalName; + assertPlan(conn, query).scanType("RANGE SCAN").serverProjectionFilterAnyOf() + .iteratorType("PARALLEL 1-WAY").table(viewIndexPhysicalName) + .keyRanges("[" + (Short.MIN_VALUE + 1) + ",'foo']").clientSortAlgo(null); } else { physicalTableName = viewIndexPhysicalName; - assertEquals(viewIndexPhysicalName, explainPlanAttributes.getTableName()); - if (saltBuckets == null) { - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(" [" + (Short.MIN_VALUE + 1) + ",'foo']", - explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); - } else { - assertEquals("PARALLEL " + saltBuckets + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals(" [X'00'," + (Short.MIN_VALUE + 1) + ",'foo'] - [" + assertPlan(conn, query).scanType("RANGE SCAN").serverProjectionFilterAnyOf() + .iteratorType("PARALLEL " + saltBuckets + "-WAY").table(viewIndexPhysicalName) + .keyRanges("[X'00'," + (Short.MIN_VALUE + 1) + ",'foo'] - [" + PVarbinary.INSTANCE.toStringLiteral(new byte[] { (byte) (saltBuckets - 1) }) + "," - + (Short.MIN_VALUE + 1) + ",'foo']", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - } + + (Short.MIN_VALUE + 1) + ",'foo']") + .clientSortAlgo("CLIENT MERGE SORT"); } return new Pair<>(physicalTableName, scan); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java index b3f28a08e93..aa306208e66 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewMetadataIT.java @@ -32,6 +32,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getPlanSteps; import static org.apache.phoenix.schema.PTable.TaskType.DROP_CHILD_VIEWS; import static org.apache.phoenix.thirdparty.com.google.common.collect.Lists.newArrayListWithExpectedSize; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; @@ -80,7 +82,6 @@ import org.apache.phoenix.schema.TableAlreadyExistsException; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.ViewUtil; @@ -235,9 +236,7 @@ public void testCreateViewMappedToExistingHbaseTableWithNSMappingEnabled() throw conn.createStatement() .execute("CREATE VIEW " + view1 + " (PK VARCHAR PRIMARY KEY, " + CF + ".COL VARCHAR)"); - assertTrue(QueryUtil - .getExplainPlan(conn.createStatement().executeQuery("explain select * from " + view1)) - .contains(NS + ":" + TBL)); + assertPlan(conn, "select * from " + view1).tableContains(NS + ":" + TBL); conn.createStatement().execute("DROP VIEW " + view1); admin.disableTable(tableName); @@ -256,9 +255,7 @@ public void testCreateViewMappedToExistingHbaseTableWithNSMappingEnabled() throw conn.createStatement() .execute("CREATE VIEW " + view2 + " (PK VARCHAR PRIMARY KEY, " + CF + ".COL VARCHAR)"); - assertTrue(QueryUtil - .getExplainPlan(conn.createStatement().executeQuery("explain select * from " + view2)) - .contains(NS + "." + TBL)); + assertPlan(conn, "select * from " + view2).tableContains(NS + "." + TBL); conn.createStatement().execute("DROP VIEW " + view2); admin.disableTable(tableName); @@ -277,9 +274,7 @@ public void testCreateViewMappedToExistingHbaseTableWithNSMappingEnabled() throw conn.createStatement() .execute("CREATE VIEW " + view3 + " (PK VARCHAR PRIMARY KEY, " + CF + ".COL VARCHAR)"); - assertTrue(QueryUtil - .getExplainPlan(conn.createStatement().executeQuery("explain select * from " + view3)) - .contains(NS + ":" + NS + "." + TBL)); + assertPlan(conn, "select * from " + view3).tableContains(NS + ":" + NS + "." + TBL); conn.createStatement().execute("DROP VIEW " + view3); admin.disableTable(tableName); @@ -947,13 +942,13 @@ private void helpTestQueryForViewOnTableThatHasIndex(Statement s1, Statement s2, // Create a index on the table s1.execute("create index " + indexName + " ON " + tableName + " (col2)"); - try (ResultSet rs = s2.executeQuery("explain select /*+ INDEX(" + viewName + " " + indexName - + ") */ * from " + viewName + " where col2 = 'aaa'")) { - String explainPlan = QueryUtil.getExplainPlan(rs); - - // check if the query uses the index - assertTrue(explainPlan.contains(indexName)); - } + String sql = "select /*+ INDEX(" + viewName + " " + indexName + ") */ * from " + viewName + + " where col2 = 'aaa'"; + // This query produces a SKIP-SCAN-JOIN where the index is scanned in a sub-plan. Assert that + // the index name appears in the plan steps. + List planSteps = getPlanSteps(s2.getConnection(), sql); + assertTrue("Expected plan to use index " + indexName + " but was: " + planSteps, + planSteps.toString().contains(indexName)); } @Test diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyIT.java index c9651f2113a..d7012009fe4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -34,8 +35,6 @@ import java.sql.Timestamp; import java.sql.Types; import org.apache.hadoop.hbase.TableName; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; @@ -770,10 +769,7 @@ private void assertBinaryValue(byte[] expected, byte[] actual) { } private void assertPointLookupsAreNotGenerated(PreparedStatement stmt) throws SQLException { - ExplainPlan explain = - stmt.unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = explain.getPlanStepsAsAttributes(); - assertEquals("FULL SCAN ", planAttributes.getExplainScanType()); + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).scanType("FULL SCAN"); } private void assertPointLookupsAreGenerated(Statement stmt, String selectSql, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyITBase.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyITBase.java index b33362e48df..40e2b6ec6b3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyITBase.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyITBase.java @@ -17,14 +17,11 @@ */ package org.apache.phoenix.end2end; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.HashMap; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.BaseTest; @@ -49,35 +46,23 @@ protected void assertPointLookupsAreGenerated(PreparedStatement stmt, int noOfPo protected void assertPointLookupsAreGenerated(QueryPlan queryPlan, int noOfPointLookups) throws SQLException { - ExplainPlan explain = queryPlan.getExplainPlan(); - ExplainPlanAttributes planAttributes = explain.getPlanStepsAsAttributes(); String expectedScanType = - "POINT LOOKUP ON " + noOfPointLookups + " KEY" + (noOfPointLookups > 1 ? "S " : " "); - assertEquals(expectedScanType, planAttributes.getExplainScanType()); + "POINT LOOKUP ON " + noOfPointLookups + " KEY" + (noOfPointLookups > 1 ? "S" : ""); + assertPlan(queryPlan.getExplainPlan().getPlanStepsAsAttributes()).scanType(expectedScanType); } protected void assertSkipScanIsGenerated(PreparedStatement stmt, int skipListSize) throws SQLException { - QueryPlan queryPlan = stmt.unwrap(PhoenixPreparedStatement.class).optimizeQuery(); - ExplainPlan explain = queryPlan.getExplainPlan(); - ExplainPlanAttributes planAttributes = explain.getPlanStepsAsAttributes(); String expectedScanType = - "SKIP SCAN ON " + skipListSize + " KEY" + (skipListSize > 1 ? "S " : " "); - assertEquals(expectedScanType, planAttributes.getExplainScanType()); + "SKIP SCAN ON " + skipListSize + " KEY" + (skipListSize > 1 ? "S" : ""); + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).scanType(expectedScanType); } protected void assertRangeScanIsGenerated(PreparedStatement stmt) throws SQLException { - QueryPlan queryPlan = stmt.unwrap(PhoenixPreparedStatement.class).optimizeQuery(); - ExplainPlan explain = queryPlan.getExplainPlan(); - ExplainPlanAttributes planAttributes = explain.getPlanStepsAsAttributes(); - String expectedScanType = "RANGE SCAN "; - assertEquals(expectedScanType, planAttributes.getExplainScanType()); + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).scanType("RANGE SCAN"); } protected void assertDegenerateScanIsGenerated(PreparedStatement stmt) throws SQLException { - QueryPlan queryPlan = stmt.unwrap(PhoenixPreparedStatement.class).optimizeQuery(); - ExplainPlan explain = queryPlan.getExplainPlan(); - ExplainPlanAttributes planAttributes = explain.getPlanStepsAsAttributes(); - assertNull(planAttributes.getExplainScanType()); + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).scanType(null); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyNullablePKIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyNullablePKIT.java index c96a985cec1..c30542b2519 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyNullablePKIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/WhereOptimizerForArrayAnyNullablePKIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -33,9 +34,6 @@ import java.sql.Types; import java.util.Arrays; import java.util.Collection; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; -import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.junit.Assume; import org.junit.Test; @@ -233,13 +231,7 @@ private String createTableAndInsertTestDataForNullablePKTests() throws Exception } private void assertQueryUsesIndex(PreparedStatement stmt, String indexName) throws SQLException { - QueryPlan queryPlan = stmt.unwrap(PhoenixPreparedStatement.class).optimizeQuery(); - ExplainPlan explain = queryPlan.getExplainPlan(); - ExplainPlanAttributes planAttributes = explain.getPlanStepsAsAttributes(); - String tableName = planAttributes.getTableName(); - System.out.println("Explain plan: " + explain.toString()); - assertTrue("Expected query to use index " + indexName + " but used table " + tableName, - tableName != null && tableName.contains(indexName)); + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).tableContains(indexName); } /** diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseImmutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseImmutableIndexIT.java index aaeeea6d07f..bb0788f4203 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseImmutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseImmutableIndexIT.java @@ -17,8 +17,8 @@ */ package org.apache.phoenix.end2end.index; -import static org.apache.phoenix.end2end.IndexToolIT.assertExplainPlan; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.getRowCount; @@ -71,7 +71,6 @@ import org.apache.phoenix.transaction.TransactionFactory; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Ignore; @@ -465,9 +464,9 @@ public void testGlobalImmutableIndexDelete() throws Exception { admin.truncateTable(TableName.valueOf(fullTableName), true); String selectFromIndex = "SELECT long_pk, varchar_pk, long_col1 FROM " + TABLE_NAME + " WHERE varchar_pk='varchar2' AND long_pk=2"; - rs = conn.createStatement().executeQuery("EXPLAIN " + selectFromIndex); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - assertExplainPlan(false, actualExplainPlan, fullTableName, fullIndexName); + // Verify the query is served by a RANGE SCAN over the index table. + assertPlan(conn, selectFromIndex).scanType("RANGE SCAN") + .tableContains(SchemaUtil.normalizeIdentifier(fullIndexName)); rs = conn.createStatement().executeQuery(selectFromIndex); assertFalse(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java index 4dca35db987..e07923ecdd0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java @@ -18,6 +18,8 @@ package org.apache.phoenix.end2end.index; import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlanWithRegions; import static org.apache.phoenix.util.TestUtil.ROW5; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; @@ -53,15 +55,12 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.ColumnResolver; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.FromCompiler; import org.apache.phoenix.end2end.CreateTableIT; import org.apache.phoenix.end2end.IndexToolIT; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.NamedTableNode; @@ -69,6 +68,7 @@ import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.explain.ExplainPlanTestUtil; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableImpl; @@ -80,7 +80,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.apache.phoenix.util.TransactionUtil; @@ -152,30 +151,19 @@ public void testIndexWithNullableFixedWithCols() throws Exception { String query = "SELECT d.char_col1, int_col1 from " + fullTableName + " as d"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlanWithRegions(conn, query).iteratorType("PARALLEL 1-WAY"); if (!uncovered) { // Optimizer would not select the uncovered index for this query - assertEquals( - columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" : "SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); + basePlan.serverProjectionFilter(columnEncoded); } - if (localIndex) { - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + basePlan.table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").scanType("RANGE SCAN"); } else if (!uncovered) { - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); + basePlan.table(fullIndexName).clientSortAlgo(null).scanType("FULL SCAN"); } - assertFalse("Explain plan regionLocation attribute should not be empty", - explainPlanAttributes.getRegionLocations().isEmpty()); + basePlan.regionLocationsNotEmpty(); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -584,26 +572,15 @@ public void testIndexWithNullableDateCol() throws Exception { String query = "SELECT" + (uncovered ? " /*+ INDEX(" + fullTableName + " " + indexName + ")*/ " : " ") + "int_pk from " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals( - columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" : "SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlanWithRegions(conn, query) + .iteratorType("PARALLEL 1-WAY").serverProjectionFilter(columnEncoded); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); + basePlan.scanType("RANGE SCAN").table(fullIndexName + "(" + fullTableName + ")") + .clientSortAlgo("CLIENT MERGE SORT").keyRanges("[1]"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } - assertFalse("Explain plan regionLocation attribute should not be empty", - explainPlanAttributes.getRegionLocations().isEmpty()); + basePlan.regionLocationsNotEmpty(); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -615,23 +592,13 @@ public void testIndexWithNullableDateCol() throws Exception { assertFalse(rs.next()); query = "SELECT date_col from " + fullTableName + " order by date_col"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals( - columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" : "SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); + basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY") + .serverProjectionFilter(columnEncoded); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); + basePlan.scanType("RANGE SCAN").table(fullIndexName + "(" + fullTableName + ")") + .clientSortAlgo("CLIENT MERGE SORT").keyRanges("[1]"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -686,23 +653,15 @@ public void testSelectAllAndAliasWithIndex() throws Exception { query = "SELECT" + (uncovered ? " /*+ INDEX(" + fullTableName + " " + indexName + ")*/" : "") + " * FROM " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlanWithRegions(conn, query).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); + basePlan.scanType("RANGE SCAN").table(fullIndexName + "(" + fullTableName + ")") + .clientSortAlgo("CLIENT MERGE SORT").keyRanges("[1]"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } - assertFalse("Explain plan regionLocation attribute should not be empty", - explainPlanAttributes.getRegionLocations().isEmpty()); + basePlan.regionLocationsNotEmpty(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -722,19 +681,9 @@ public void testSelectAllAndAliasWithIndex() throws Exception { assertFalse(rs.next()); query = "SELECT v1 as foo FROM " + fullTableName + " WHERE v2 = '1' ORDER BY foo"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("[\"V1\"]", explainPlanAttributes.getServerSortedBy()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - if (localIndex) { - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - } else { - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - } + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .serverSortedBy("[\"V1\"]").clientSortAlgo("CLIENT MERGE SORT") + .table(localIndex ? fullIndexName + "(" + fullTableName + ")" : fullIndexName); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -784,29 +733,19 @@ public void testSelectCF() throws Exception { conn.commit(); query = "SELECT * FROM " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullTableName, explainPlanAttributes.getTableName()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(fullTableName); query = "SELECT" + (uncovered ? " /*+ INDEX(" + fullTableName + " " + indexName + ")*/ " : " ") + "a.* FROM " + fullTableName; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN").table(fullIndexName + "(" + fullTableName + ")") + .keyRanges("[1]").clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); + basePlan.scanType("FULL SCAN").table(fullIndexName); } rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -970,23 +909,15 @@ public void testMultipleUpdatesAcrossRegions() throws Exception { // make sure the index is working as expected query = "SELECT" + (uncovered ? " /*+ INDEX(" + testTable + " " + indexName + ")*/ " : " ") + "* FROM " + testTable; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals( - columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" : "SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).serverProjectionFilter(columnEncoded); if (localIndex) { - assertEquals("PARALLEL 2-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + testTable + ")", explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.iteratorType("PARALLEL 2-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + testTable + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(fullIndexName) + .clientSortAlgo(null); } // check that the data table matches as expected @@ -1048,18 +979,12 @@ public void testIndexWithCaseSensitiveCols() throws Exception { } query = "SELECT * FROM " + fullTableName + " WHERE \"v2\" = '1'"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN"); if (localIndex) { - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,'1']", explainPlanAttributes.getKeyRanges()); + basePlan.table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1,'1']"); } else { - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertEquals(" ['1']", explainPlanAttributes.getKeyRanges()); + basePlan.table(fullIndexName).keyRanges("['1']"); } rs = conn.createStatement().executeQuery(query); @@ -1086,20 +1011,12 @@ public void testIndexWithCaseSensitiveCols() throws Exception { query = "SELECT \"V1\", \"V1\" as foo1, \"v2\" as foo, \"v2\" as \"Foo1\", \"v2\" FROM " + fullTableName + " ORDER BY foo"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN").table(fullIndexName + "(" + fullTableName + ")") + .keyRanges("[1]").clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -1220,20 +1137,13 @@ private void testIndexWithDecimalCol(boolean enableServerSideUpsert) throws Exce query = "SELECT" + (uncovered ? " /*+ INDEX(" + fullTableName + " " + indexName + ")*/ " : " ") + "decimal_pk, decimal_col1, decimal_col2 from " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + fullTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN").table(fullIndexName + "(" + fullTableName + ")") + .keyRanges("[1]").clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -1547,12 +1457,11 @@ public void testSelectUncoveredWithCoveredField() throws Exception { query = "SELECT /*+ INDEX(" + fullTableName + " " + indexName + ")*/ " + columns + " from " + fullTableName + " where int_col1=2 and long_col1=2"; - rs = stmt.executeQuery("Explain " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals("bad plan with columns:" + columns, "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + fullIndexName + " [2]\n" - + " SERVER MERGE [A.VARCHAR_COL1, A.CHAR_COL1, A.DECIMAL_COL1, A.DATE1, B.VARCHAR_COL2, B.CHAR_COL2, B.INT_COL2, B.DECIMAL_COL2, B.DATE2]\n" - + " SERVER FILTER BY A.\"LONG_COL1\" = 2", explainPlan); + assertPlan(conn, query).scanType("RANGE SCAN").tableContains(fullIndexName).keyRanges("[2]") + .serverMergeColumns( + "[A.VARCHAR_COL1, A.CHAR_COL1, A.DECIMAL_COL1, A.DATE1, B.VARCHAR_COL2, B.CHAR_COL2," + + " B.INT_COL2, B.DECIMAL_COL2, B.DATE2]") + .serverWhereFilter("SERVER FILTER BY A.\"LONG_COL1\" = 2"); rs = stmt.executeQuery(query); assertTrue(rs.next()); // Test the projector thoroughly diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java index 329cb4df55e..c3c8220accf 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end.index; import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.ROW5; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; @@ -53,15 +54,12 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.ColumnResolver; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.FromCompiler; import org.apache.phoenix.end2end.CreateTableIT; import org.apache.phoenix.end2end.IndexToolIT; import org.apache.phoenix.end2end.ParallelStatsDisabledWithRegionMovesIT; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.NamedTableNode; @@ -69,6 +67,7 @@ import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.explain.ExplainPlanTestUtil; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableImpl; @@ -80,7 +79,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.apache.phoenix.util.TransactionUtil; @@ -168,26 +166,17 @@ public void testIndexWithNullableFixedWithCols() throws Exception { String query = "SELECT d.char_col1, int_col1 from " + fullTableName + " as d"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY"); if (!uncovered) { // Optimizer would not select the uncovered index for this query - assertEquals( - columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" : "SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); + basePlan.serverProjectionFilter(columnEncoded); } - if (localIndex) { - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + basePlan.table(fullTableName).keyRanges("[1]").clientSortAlgo("CLIENT MERGE SORT") + .scanType("RANGE SCAN"); } else if (!uncovered) { - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); + basePlan.table(fullIndexName).clientSortAlgo(null).scanType("FULL SCAN"); } ResultSet rs = conn.createStatement().executeQuery(query); @@ -642,22 +631,13 @@ public void testIndexWithNullableDateCol() throws Exception { String query = "SELECT" + (uncovered ? " /*+ INDEX(" + fullTableName + " " + indexName + ")*/ " : " ") + "int_pk from " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals( - columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" : "SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, query) + .iteratorType("PARALLEL 1-WAY").serverProjectionFilter(columnEncoded); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); + basePlan.scanType("RANGE SCAN").table(fullTableName).clientSortAlgo("CLIENT MERGE SORT") + .keyRanges("[1]"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } ResultSet rs = conn.createStatement().executeQuery(query); @@ -674,22 +654,13 @@ public void testIndexWithNullableDateCol() throws Exception { assertFalse(rs.next()); query = "SELECT date_col from " + fullTableName + " order by date_col"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals( - columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" : "SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); + basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY") + .serverProjectionFilter(columnEncoded); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); + basePlan.scanType("RANGE SCAN").table(fullTableName).clientSortAlgo("CLIENT MERGE SORT") + .keyRanges("[1]"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -750,19 +721,13 @@ public void testSelectAllAndAliasWithIndex() throws Exception { query = "SELECT" + (uncovered ? " /*+ INDEX(" + fullTableName + " " + indexName + ")*/" : "") + " * FROM " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); + basePlan.scanType("RANGE SCAN").table(fullTableName).clientSortAlgo("CLIENT MERGE SORT") + .keyRanges("[1]"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -787,18 +752,9 @@ public void testSelectAllAndAliasWithIndex() throws Exception { assertFalse(rs.next()); query = "SELECT v1 as foo FROM " + fullTableName + " WHERE v2 = '1' ORDER BY foo"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("[\"V1\"]", explainPlanAttributes.getServerSortedBy()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - if (localIndex) { - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - } else { - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - } + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .serverSortedBy("[\"V1\"]").clientSortAlgo("CLIENT MERGE SORT") + .table(localIndex ? fullTableName : fullIndexName); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -852,28 +808,19 @@ public void testSelectCF() throws Exception { conn.commit(); query = "SELECT * FROM " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullTableName, explainPlanAttributes.getTableName()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(fullTableName); query = "SELECT" + (uncovered ? " /*+ INDEX(" + fullTableName + " " + indexName + ")*/ " : " ") + "a.* FROM " + fullTableName; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN").table(fullTableName).keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); + basePlan.scanType("FULL SCAN").table(fullIndexName); } rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -1055,23 +1002,14 @@ public void testMultipleUpdatesAcrossRegions() throws Exception { // make sure the index is working as expected query = "SELECT" + (uncovered ? " /*+ INDEX(" + testTable + " " + indexName + ")*/ " : " ") + "* FROM " + testTable; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals( - columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" : "SERVER FILTER BY EMPTY COLUMN ONLY", - explainPlanAttributes.getServerWhereFilter()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).serverProjectionFilter(columnEncoded); if (localIndex) { - assertEquals("PARALLEL 2-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(testTable, explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.iteratorType("PARALLEL 2-WAY").scanType("RANGE SCAN").table(testTable) + .keyRanges("[1]").clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(fullIndexName) + .clientSortAlgo(null); } // check that the data table matches as expected @@ -1139,17 +1077,12 @@ public void testIndexWithCaseSensitiveCols() throws Exception { } query = "SELECT * FROM " + fullTableName + " WHERE \"v2\" = '1'"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN"); if (localIndex) { - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - assertEquals(" [1,'1']", explainPlanAttributes.getKeyRanges()); + basePlan.table(fullTableName).keyRanges("[1,'1']"); } else { - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertEquals(" ['1']", explainPlanAttributes.getKeyRanges()); + basePlan.table(fullIndexName).keyRanges("['1']"); } rs = conn.createStatement().executeQuery(query); @@ -1178,19 +1111,12 @@ public void testIndexWithCaseSensitiveCols() throws Exception { query = "SELECT \"V1\", \"V1\" as foo1, \"v2\" as foo, \"v2\" as \"Foo1\", \"v2\" FROM " + fullTableName + " ORDER BY foo"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN").table(fullTableName).keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -1320,19 +1246,13 @@ private void testIndexWithDecimalCol(boolean enableServerSideUpsert) throws Exce query = "SELECT" + (uncovered ? " /*+ INDEX(" + fullTableName + " " + indexName + ")*/ " : " ") + "decimal_pk, decimal_col1, decimal_col2 from " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullTableName, explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN").table(fullTableName).keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(fullIndexName).clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -1673,14 +1593,11 @@ public void testSelectUncoveredWithCoveredField() throws Exception { query = "SELECT /*+ INDEX(" + fullTableName + " " + indexName + ")*/ " + columns + " from " + fullTableName + " where int_col1=2 and long_col1=2"; - rs = stmt.executeQuery("Explain " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals("bad plan with columns:" + columns, - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + " [2]\n" - + " SERVER MERGE [A.VARCHAR_COL1, A.CHAR_COL1, A.DECIMAL_COL1," - + " A.DATE1, B.VARCHAR_COL2, B.CHAR_COL2, B.INT_COL2, " + "B.DECIMAL_COL2, B.DATE2]\n" - + " SERVER FILTER BY A.\"LONG_COL1\" = 2", - explainPlan); + assertPlan(conn, query).scanType("RANGE SCAN").tableContains(fullIndexName).keyRanges("[2]") + .serverMergeColumns( + "[A.VARCHAR_COL1, A.CHAR_COL1, A.DECIMAL_COL1, A.DATE1, B.VARCHAR_COL2, B.CHAR_COL2," + + " B.INT_COL2, B.DECIMAL_COL2, B.DATE2]") + .serverWhereFilter("SERVER FILTER BY A.\"LONG_COL1\" = 2"); rs = stmt.executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(fullTableName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ChildViewsUseParentViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ChildViewsUseParentViewIndexIT.java index 9b84aad8f79..cf2f43c005d 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ChildViewsUseParentViewIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ChildViewsUseParentViewIndexIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -28,12 +29,10 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.schema.PTable; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -165,16 +164,12 @@ private void assertQueryUsesIndex(final String baseTableName, final String viewN Connection conn, boolean isChildView) throws SQLException { String sql = "SELECT A0, A1, A2, A4 FROM " + viewName + " WHERE A4 IN ('1', '2', '3') ORDER BY A4, A2"; - ExplainPlan plan = conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SKIP SCAN ON 3 KEYS ", explainPlanAttributes.getExplainScanType()); - assertEquals("_IDX_" + baseTableName, explainPlanAttributes.getTableName()); String childViewScanKey = isChildView ? ",'Y'" : ""; - assertEquals(" [" + Short.MIN_VALUE + ",'1'" + childViewScanKey + "] - [" + Short.MIN_VALUE - + ",'3'" + childViewScanKey + "]", explainPlanAttributes.getKeyRanges()); + assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY").serverFirstKeyOnlyProjection(true) + .scanType("SKIP SCAN ON 3 KEYS").table("_IDX_" + baseTableName) + .keyRanges("[" + Short.MIN_VALUE + ",'1'" + childViewScanKey + "] - [" + Short.MIN_VALUE + + ",'3'" + childViewScanKey + "]") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); ResultSet rs = conn.createStatement().executeQuery(sql); assertTrue(rs.next()); @@ -193,15 +188,11 @@ private void assertQueryUsesIndex(final String baseTableName, final String viewN private void assertQueryUsesBaseTable(final String baseTableName, final String viewName, Connection conn) throws SQLException { String sql = "SELECT A0, A1, A2, A4 FROM " + viewName + " WHERE A4 IN ('1', '2', '3') "; - ExplainPlan plan = conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals( - "SERVER FILTER BY (A4 IN ('1','2','3') AND ((A1 = 'X' AND A2 = 'Y') AND A3 = 'Z'))", - explainPlanAttributes.getServerWhereFilter()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(baseTableName, explainPlanAttributes.getTableName()); + assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY") + .serverWhereFilter( + "SERVER FILTER BY (A4 IN ('1','2','3') AND ((A1 = 'X' AND A2 = 'Y') AND A3 = 'Z'))") + .scanType("FULL SCAN").table(baseTableName).indexRule(OptimizerReasons.RULE_DATA_TABLE) + .indexRejectedNone(); ResultSet rs = conn.createStatement().executeQuery(sql); assertTrue(rs.next()); @@ -273,18 +264,11 @@ private void assertQueryIndex(String viewName, String baseTableName, Connection String sql = "SELECT WO_ID FROM " + viewName + " WHERE WO_ID IN ('003xxxxxxxxxxx1', '003xxxxxxxxxxx2', '003xxxxxxxxxxx3', '003xxxxxxxxxxx4', '003xxxxxxxxxxx5') " + " AND (A_DATE > TO_DATE('2016-01-01 06:00:00.0')) " + " ORDER BY WO_ID, A_DATE DESC"; - ExplainPlan plan = conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SKIP SCAN ON 5 RANGES ", explainPlanAttributes.getExplainScanType()); - assertEquals("_IDX_" + baseTableName, explainPlanAttributes.getTableName()); - assertEquals( - " [" + Short.MIN_VALUE + ",'00Dxxxxxxxxxxx1','003xxxxxxxxxxx1',*] - [" + Short.MIN_VALUE - + ",'00Dxxxxxxxxxxx1','003xxxxxxxxxxx5',~'2016-01-01 06:00:00.000']", - explainPlanAttributes.getKeyRanges()); - + assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY").serverFirstKeyOnlyProjection(true) + .scanType("SKIP SCAN ON 5 RANGES").table("_IDX_" + baseTableName) + .keyRanges("[" + Short.MIN_VALUE + ",'00Dxxxxxxxxxxx1','003xxxxxxxxxxx1',*] - [" + + Short.MIN_VALUE + ",'00Dxxxxxxxxxxx1','003xxxxxxxxxxx5',~'2016-01-01 06:00:00.000']") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); ResultSet rs = conn.createStatement().executeQuery(sql); for (int i = 0; i < expectedRows; ++i) { assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java index a862df07adf..2d5de2dbed5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java @@ -74,6 +74,11 @@ protected void waitForEventualConsistency() throws Exception { Thread.sleep(18000); } + @Override + protected boolean isEventualConsistency() { + return true; + } + @Parameterized.Parameters(name = "async={0},encoded={1}") public static synchronized Collection data() { List list = Lists.newArrayListWithExpectedSize(4); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java index d7b5a35e1c6..a54d5903be8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java @@ -74,6 +74,11 @@ protected void waitForEventualConsistency() throws Exception { Thread.sleep(15000); } + @Override + protected boolean isEventualConsistency() { + return true; + } + @Parameterized.Parameters(name = "async={0},encoded={1}") public static synchronized Collection data() { List list = Lists.newArrayListWithExpectedSize(4); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java index 8d7d4c546ad..1c6384f22c7 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java @@ -29,9 +29,12 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_VALID_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getExplainAttributes; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -52,19 +55,20 @@ import java.util.Properties; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.mapreduce.CounterGroup; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.IndexToolIT; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.hbase.index.IndexRegionObserver; -import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.mapreduce.index.IndexTool; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.After; import org.junit.Assume; @@ -140,19 +144,49 @@ public void unsetFailForTesting() throws Exception { public static void assertExplainPlan(Connection conn, String selectSql, String dataTableFullName, String indexTableFullName) throws SQLException { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - IndexToolIT.assertExplainPlan(false, actualExplainPlan, dataTableFullName, indexTableFullName); + assertExplainPlan(conn, selectSql, dataTableFullName, indexTableFullName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS); + } + + public static void assertExplainPlan(Connection conn, String selectSql, String dataTableFullName, + String indexTableFullName, String expectedRule) throws SQLException { + // Verify the query is served by a RANGE SCAN over the index table not the data table. + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN").indexRule(expectedRule); + assertScannedTableIsIndex(attributes, indexTableFullName); + } + + /** + * Asserts the selection {@code expectedRule} and the separate + * {@code "matches "} functional index match. + */ + public static void assertExplainPlan(Connection conn, String selectSql, String dataTableFullName, + String indexTableFullName, String expectedRule, String expectedFunctionalMatchExpr) + throws SQLException { + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN").indexRule(expectedRule) + .functionalMatch(expectedFunctionalMatchExpr); + assertScannedTableIsIndex(attributes, indexTableFullName); + } + + private static void assertScannedTableIsIndex(ExplainPlanAttributes attributes, + String indexTableFullName) { + String actualTable = + attributes.getTableName() == null ? null : attributes.getTableName().replaceAll(":", "."); + String expectedTable = SchemaUtil.normalizeIdentifier(indexTableFullName); + assertTrue("expected scanned table <" + actualTable + "> to use index <" + expectedTable + ">", + actualTable != null && actualTable.contains(expectedTable)); } public static void assertExplainPlanWithLimit(Connection conn, String selectSql, String dataTableFullName, String indexTableFullName, int limit) throws SQLException { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - IndexToolIT.assertExplainPlan(false, actualExplainPlan, dataTableFullName, indexTableFullName); - String expectedLimitPlan = String.format("SERVER %d ROW LIMIT", limit); - assertTrue(actualExplainPlan + "\n expected to contain \n" + expectedLimitPlan, - actualExplainPlan.contains(expectedLimitPlan)); + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN").serverRowLimit((long) limit); + String actualTable = + attributes.getTableName() == null ? null : attributes.getTableName().replaceAll(":", "."); + String expectedTable = SchemaUtil.normalizeIdentifier(indexTableFullName); + assertTrue("expected scanned table <" + actualTable + "> to use index <" + expectedTable + ">", + actualTable != null && actualTable.contains(expectedTable)); } private void populateTable(String tableName) throws Exception { @@ -233,7 +267,8 @@ public void testPhoenixRowTimestamp() throws Exception { + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -252,7 +287,8 @@ public void testPhoenixRowTimestamp() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -265,7 +301,8 @@ public void testPhoenixRowTimestamp() throws Exception { + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -277,10 +314,15 @@ public void testPhoenixRowTimestamp() throws Exception { String noIndexQuery = "SELECT /*+ NO_INDEX */ val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; - // Verify that we will read from the data table - rs = conn.createStatement().executeQuery("EXPLAIN " + noIndexQuery); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(explainPlan.contains("FULL SCAN OVER " + dataTableName)); + // Verify that we will read from the data table. The NO_INDEX hint rejects every + // secondary index candidate. Under STRONG consistency only the user index exists. + // Under EVENTUAL consistency the user index is paired with an auto-created CDC index, + // so two candidates are rejected. Match the user index by name rather than position + // since the rejection order is not guaranteed. + int expectedRejected = isEventualConsistency() ? 2 : 1; + assertPlan(conn, noIndexQuery).scanType("FULL SCAN").table(dataTableName) + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedCount(expectedRejected) + .indexRejectedContains(indexTableName, OptimizerReasons.REASON_EXCLUDED_BY_NO_INDEX_HINT); rs = conn.createStatement().executeQuery(noIndexQuery); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -300,7 +342,8 @@ public void testPhoenixRowTimestamp() throws Exception { query = "SELECT val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -330,7 +373,8 @@ public void testPhoenixRowTimestamp() throws Exception { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("ab", rs.getString(1)); @@ -367,7 +411,10 @@ public void testPhoenixRowTimestamp() throws Exception { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; - assertExplainPlan(conn, query, dataTableName, indexTableName); + // This query carries an explicit INDEX hint, so the selection rule is "hint". The functional + // index match over PHOENIX_ROW_TIMESTAMP() is disclosed separately. + assertExplainPlan(conn, query, dataTableName, indexTableName, OptimizerReasons.RULE_HINT, + "PHOENIX_ROW_TIMESTAMP()"); waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -739,10 +786,9 @@ public void testPartialRowUpdateForImmutable() throws Exception { // now read the same row from data table selectSql = "SELECT * from " + dataTableName + " WHERE id = 'a'"; + assertPlan(conn, selectSql).table(dataTableName).indexRule(OptimizerReasons.RULE_POINT_LOOKUP) + .indexRejectedNone(); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(dataTableName)); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); assertEquals("ab", rs.getString(2)); @@ -1264,11 +1310,7 @@ public void testUnverifiedIndexRowWithSkipScanFilter() throws Exception { String selectSql = "SELECT id, val1, val3 from " + dataTableName + " WHERE val1 IN ('ab', 'bcc') "; // Verify that we will read from the index table - try (ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql)) { - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - String expectedExplainPlan = String.format("SKIP SCAN ON 2 KEYS OVER %s", indexName); - assertTrue(actualExplainPlan.contains(expectedExplainPlan)); - } + assertPlan(conn, selectSql).scanType("SKIP SCAN ON 2 KEYS").table(indexName); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { assertTrue(rs.next()); assertEquals("a", rs.getString("id")); @@ -1289,16 +1331,11 @@ public void testUnverifiedIndexRowWithSkipScanFilter() throws Exception { selectSql = "SELECT id, val3 from " + dataTableName + " WHERE val1 IN ('bc') AND val2 IN ('bcd', 'xcdf') AND val3 = 'bcde' "; - // Verify that we will read from the index table - try (ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql)) { - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - String expectedExplainPlan = String.format("SKIP SCAN ON 2 KEYS OVER %s", indexName); - String filter = "SERVER FILTER BY"; - assertTrue(String.format("actual=%s", actualExplainPlan), - actualExplainPlan.contains(expectedExplainPlan)); - assertTrue(String.format("actual=%s", actualExplainPlan), - actualExplainPlan.contains(filter)); - } + // Verify that we will read from the index table with a server filter + ExplainPlanAttributes skipScanAttributes = getExplainAttributes(conn, selectSql); + assertPlan(skipScanAttributes).scanType("SKIP SCAN ON 2 KEYS").table(indexName); + assertNotNull("expected a server filter, plan=" + skipScanAttributes, + skipScanAttributes.getServerWhereFilter()); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { assertTrue(rs.next()); assertEquals("b", rs.getString("id")); @@ -1351,9 +1388,7 @@ public void testUnverifiedIndexRowWithSkipScanFilter2() throws Exception { List expectedIDs = Lists.newArrayList("a", "d"); List actualIDs = Lists.newArrayList(); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String actualExplainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(actualExplainPlan.contains(indexName)); + assertPlan(conn, selectSql).table(indexName); while (rs.next()) { actualIDs.add(rs.getString("id")); } @@ -1367,9 +1402,7 @@ public void testUnverifiedIndexRowWithSkipScanFilter2() throws Exception { expectedIDs = Lists.newArrayList("e", "g"); actualIDs = Lists.newArrayList(); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String actualExplainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(actualExplainPlan.contains(indexName)); + assertPlan(conn, selectSql).table(indexName); while (rs.next()) { actualIDs.add(rs.getString("id")); } @@ -1532,11 +1565,11 @@ public void testUncoveredIndexWithDistinctPrefixFilter() throws Exception { private void verifyDistinctQueryOnIndex(Connection conn, String indexName, String query, List expectedValues) throws SQLException, IOException { try (ResultSet rs = conn.createStatement().executeQuery(query)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String actualExplainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(actualExplainPlan.contains(indexName)); - assertTrue(actualExplainPlan, actualExplainPlan.contains("SERVER DISTINCT PREFIX FILTER")); - List actualValues = Lists.newArrayList(); + ExplainPlanAttributes attributes = getExplainAttributes(conn, query); + assertPlan(attributes).table(indexName); + assertNotNull("expected a server distinct prefix filter, plan=" + attributes, + attributes.getServerDistinctFilter()); + List actualValues = Lists.newArrayList(); while (rs.next()) { actualValues.add(rs.getString(1)); } @@ -1572,17 +1605,12 @@ public void testUnverifiedIndexRowWithFirstKeyOnlyFilter() throws Exception { String selectSql = "SELECT id, val3 from " + dataTableName + " WHERE val1 = 'bc' and val2 = 'bcd' "; - // Verify that we will read from the index table - try (ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql)) { - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - String expectedExplainPlan = String.format("RANGE SCAN OVER %s", indexName); - String filter = - String.format("SERVER FILTER BY %s ONLY AND", encoded ? "FIRST KEY" : "EMPTY COLUMN"); - assertTrue(String.format("actual=%s", actualExplainPlan), - actualExplainPlan.contains(expectedExplainPlan)); - assertTrue(String.format("actual=%s", actualExplainPlan), - actualExplainPlan.contains(filter)); - } + // Verify that we will read from the index table with a first-key-only/empty-column filter + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN").table(indexName) + .serverProjectionFilter(encoded); + assertNotNull("serverWhereFilter=" + attributes.getServerWhereFilter(), + attributes.getServerWhereFilter()); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { assertTrue(rs.next()); assertEquals("b", rs.getString("id")); @@ -1617,19 +1645,15 @@ public void testIndexRowWithNullIncludedColumnAndFilter() throws Exception { String dql = String.format("select id, val2 from %s where val1='ab' and val3='abcd'", dataTableName); + assertPlan(conn, dql).table(indexName); try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexName)); assertFalse(rs.next()); } dql = String.format("select id, val2 from %s where val1='ab' and val3 is null", dataTableName); + assertPlan(conn, dql).table(indexName); try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexName)); assertTrue(rs.next()); assertEquals("abc", rs.getString("val2")); } @@ -1642,10 +1666,8 @@ public void testIndexRowWithNullIncludedColumnAndFilter() throws Exception { dql = String.format("select id, val2 from %s where val1='ac' and val3 is null", dataTableName); + assertPlan(conn, dql).table(indexName); try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexName)); assertTrue(rs.next()); assertNull(rs.getString("val2")); } @@ -1730,10 +1752,8 @@ public void testIndexToolWithMultipleDeleteFamilyMarkers() throws Exception { // delete family marker on the unverified index row String dql = String.format("select id, val2, val3 from %s where val1='ab'", dataTableName); + assertPlan(conn, dql).table(indexName); try (ResultSet rs = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexName)); assertFalse(rs.next()); } } @@ -1884,11 +1904,11 @@ public void testWithDistinctPrefixFilter() throws Exception { waitForEventualConsistency(); String distinctQuery = "SELECT DISTINCT val1 FROM " + dataTableName; try (ResultSet rs = conn.createStatement().executeQuery(distinctQuery)) { - PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(indexTableName)); - assertTrue(explainPlan.contains("SERVER DISTINCT PREFIX FILTER OVER")); - List actualValues = Lists.newArrayList(); + ExplainPlanAttributes attributes = getExplainAttributes(conn, distinctQuery); + assertPlan(attributes).table(indexTableName); + assertNotNull("expected a server distinct prefix filter, plan=" + attributes, + attributes.getServerDistinctFilter()); + List actualValues = Lists.newArrayList(); while (rs.next()) { actualValues.add(rs.getString(1)); } @@ -1912,6 +1932,10 @@ public static void commitWithException(Connection conn) { protected void waitForEventualConsistency() throws Exception { } + protected boolean isEventualConsistency() { + return false; + } + protected void verifyTableHealth(Connection conn, String dataTableName, String indexTableName) throws Exception { // Add two rows and check everything is still okay diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerWithRegionMovesIT.java index 6583df2d84b..3fbc8f4a626 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerWithRegionMovesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerWithRegionMovesIT.java @@ -30,9 +30,12 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_VALID_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.getExplainAttributes; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -58,6 +61,7 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionStatesCount; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.end2end.IndexToolIT; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; @@ -70,8 +74,8 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.After; import org.junit.Before; @@ -251,19 +255,25 @@ public void unsetFailForTesting() throws Exception { public static void assertExplainPlan(Connection conn, String selectSql, String dataTableFullName, String indexTableFullName) throws SQLException { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - IndexToolIT.assertExplainPlan(false, actualExplainPlan, dataTableFullName, indexTableFullName); + // Verify the query is served by a RANGE SCAN over the index table and not the data table. + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN"); + String actualTable = + attributes.getTableName() == null ? null : attributes.getTableName().replaceAll(":", "."); + String expectedTable = SchemaUtil.normalizeIdentifier(indexTableFullName); + assertTrue("expected scanned table <" + actualTable + "> to use index <" + expectedTable + ">", + actualTable != null && actualTable.contains(expectedTable)); } public static void assertExplainPlanWithLimit(Connection conn, String selectSql, String dataTableFullName, String indexTableFullName, int limit) throws SQLException { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - IndexToolIT.assertExplainPlan(false, actualExplainPlan, dataTableFullName, indexTableFullName); - String expectedLimitPlan = String.format("SERVER %d ROW LIMIT", limit); - assertTrue(actualExplainPlan + "\n expected to contain \n" + expectedLimitPlan, - actualExplainPlan.contains(expectedLimitPlan)); + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN").serverRowLimit((long) limit); + String actualTable = + attributes.getTableName() == null ? null : attributes.getTableName().replaceAll(":", "."); + String expectedTable = SchemaUtil.normalizeIdentifier(indexTableFullName); + assertTrue("expected scanned table <" + actualTable + "> to use index <" + expectedTable + ">", + actualTable != null && actualTable.contains(expectedTable)); } private void populateTable(String tableName) throws Exception { @@ -398,9 +408,7 @@ public void testPhoenixRowTimestamp() throws Exception { + dataTableName + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the data table - rs = conn.createStatement().executeQuery("EXPLAIN " + noIndexQuery); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(explainPlan.contains("FULL SCAN OVER " + dataTableName)); + assertPlan(conn, noIndexQuery).scanType("FULL SCAN").table(dataTableName); rs = conn.createStatement().executeQuery(noIndexQuery); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -1447,11 +1455,7 @@ public void testUnverifiedIndexRowWithSkipScanFilter() throws Exception { String selectSql = "SELECT id, val1, val3 from " + dataTableName + " WHERE val1 IN ('ab', 'bcc') "; // Verify that we will read from the index table - try (ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql)) { - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - String expectedExplainPlan = String.format("SKIP SCAN ON 2 KEYS OVER %s", indexName); - assertTrue(actualExplainPlan.contains(expectedExplainPlan)); - } + assertPlan(conn, selectSql).scanType("SKIP SCAN ON 2 KEYS").table(indexName); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { assertTrue(rs.next()); assertEquals("a", rs.getString("id")); @@ -1474,16 +1478,11 @@ public void testUnverifiedIndexRowWithSkipScanFilter() throws Exception { selectSql = "SELECT id, val3 from " + dataTableName + " WHERE val1 IN ('bc') AND val2 IN ('bcd', 'xcdf') AND val3 = 'bcde' "; - // Verify that we will read from the index table - try (ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql)) { - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - String expectedExplainPlan = String.format("SKIP SCAN ON 2 KEYS OVER %s", indexName); - String filter = "SERVER FILTER BY"; - assertTrue(String.format("actual=%s", actualExplainPlan), - actualExplainPlan.contains(expectedExplainPlan)); - assertTrue(String.format("actual=%s", actualExplainPlan), - actualExplainPlan.contains(filter)); - } + // Verify that we will read from the index table with a server filter + ExplainPlanAttributes skipScanAttributes = getExplainAttributes(conn, selectSql); + assertPlan(skipScanAttributes).scanType("SKIP SCAN ON 2 KEYS").table(indexName); + assertNotNull("expected a server filter, plan=" + skipScanAttributes, + skipScanAttributes.getServerWhereFilter()); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -1525,17 +1524,12 @@ public void testUnverifiedIndexRowWithFirstKeyOnlyFilter() throws Exception { String selectSql = "SELECT id, val3 from " + dataTableName + " WHERE val1 = 'bc' and val2 = 'bcd' "; - // Verify that we will read from the index table - try (ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql)) { - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - String expectedExplainPlan = String.format("RANGE SCAN OVER %s", indexName); - String filter = - String.format("SERVER FILTER BY %s ONLY AND", encoded ? "FIRST KEY" : "EMPTY COLUMN"); - assertTrue(String.format("actual=%s", actualExplainPlan), - actualExplainPlan.contains(expectedExplainPlan)); - assertTrue(String.format("actual=%s", actualExplainPlan), - actualExplainPlan.contains(filter)); - } + // Verify that we will read from the index table with a first-key-only/empty-column filter + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN").table(indexName) + .serverProjectionFilter(encoded); + assertNotNull("serverWhereFilter=" + attributes.getServerWhereFilter(), + attributes.getServerWhereFilter()); try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) { assertTrue(rs.next()); moveRegionsOfTable(dataTableName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java index fe7a1e8e4b8..3d34c5b4203 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertMutationPlan; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -28,8 +30,9 @@ import java.util.regex.Pattern; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; +import org.apache.phoenix.optimize.OptimizerReasons; +import org.apache.phoenix.query.explain.ExplainPlanTestUtil.ExplainPlanAssert; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -78,13 +81,13 @@ public void testIndexDeleteOptimizationWithLocalIndex() throws Exception { + " SELECT TO_CHAR(rand()*100),rand()*10000,rand()*10000,rand()*10000,TO_CHAR(rand()*100) FROM " + dataTableName); } - ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - String expected = "DELETE ROWS CLIENT SELECT\n" + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + indexTableName + "L" + "(" + dataTableName + ") [1,*] - [1,100]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT"; - String actual = QueryUtil.getExplainPlan(rs); - assertEquals(expected, actual); - rs = conn1.createStatement().executeQuery("SELECT COUNT(*) FROM " + dataTableName); + assertMutationPlan(conn1, query).abstractExplainPlan("DELETE ROWS CLIENT SELECT") + .scanType("RANGE SCAN").table(indexTableName + "L(" + dataTableName + ")") + .keyRanges("[1,*] - [1,100]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedCount(1) + .indexRejected(0, indexTableName + "G", OptimizerReasons.REASON_NO_PK_PREFIX_BOUND); + ResultSet rs = conn1.createStatement().executeQuery("SELECT COUNT(*) FROM " + dataTableName); rs.next(); int count = rs.getInt(1); int deleted = conn1.createStatement().executeUpdate(query); @@ -153,14 +156,11 @@ private void testOptimization(String dataTableName, String dataTableFullName, String query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ * FROM " + dataTableName + " where v1='a'"; - ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - String expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " ['a']\n" - + " SERVER MERGE [0.K3]\n" + " SERVER FILTER BY FIRST KEY ONLY"; - String actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); + assertPlan(conn1, query).scanType("RANGE SCAN").table(indexTableName).keyRanges("['a']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_HINT).indexRejectedNone(); - rs = conn1.createStatement().executeQuery(query); + ResultSet rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("f", rs.getString("t_id")); assertEquals(1, rs.getInt("k1")); @@ -175,12 +175,9 @@ private void testOptimization(String dataTableName, String dataTableFullName, query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ * FROM " + dataTableName + " where v1='a'"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " ['a']\n" - + " SERVER MERGE [0.K3]\n" + " SERVER FILTER BY FIRST KEY ONLY"; - actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); + assertPlan(conn1, query).scanType("RANGE SCAN").table(indexTableName).keyRanges("['a']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_HINT).indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -199,13 +196,10 @@ private void testOptimization(String dataTableName, String dataTableFullName, query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ * FROM " + dataTableName + " where v1='a' limit 1"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - expected = "CLIENT SERIAL 1-WAY RANGE SCAN OVER " + indexTableName + " ['a']\n" - + " SERVER MERGE [0.K3]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER 1 ROW LIMIT\n" + "CLIENT 1 ROW LIMIT"; - actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); + assertPlan(conn1, query).iteratorType("SERIAL").scanType("RANGE SCAN").table(indexTableName) + .keyRanges("['a']").serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .serverRowLimit(1L).clientRowLimit(1).indexRule(OptimizerReasons.RULE_HINT) + .indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -219,12 +213,10 @@ private void testOptimization(String dataTableName, String dataTableFullName, query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " where v1<='z' and k3 > 1 order by V1,t_id"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n" - + " SERVER MERGE [0.K3]\n" + " SERVER FILTER BY FIRST KEY ONLY AND \"K3\" > 1"; - actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); + assertPlan(conn1, query).scanType("RANGE SCAN").table(indexTableName).keyRanges("[*] - ['z']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY \"K3\" > 1").indexRule(OptimizerReasons.RULE_HINT) + .indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -251,18 +243,27 @@ private void testOptimization(String dataTableName, String dataTableFullName, query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + "), NO_INDEX_SERVER_MERGE */ t_id, k1, k2, k3, V1 from " + dataTableFullName + " where v1<='z' and k3 > 1 order by V1,t_id"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - expected = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dataTableName + "\n" - + " SERVER FILTER BY K3 > 1\n" + " SERVER SORTED BY \\[" + dataTableName + "\\.V1, " - + dataTableName + "\\.T_ID\\]\n" + "CLIENT MERGE SORT\n" + " SKIP-SCAN-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName - + " \\[\\*\\] - \\['z'\\]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + "\\.T_ID\", \"" + dataTableName - + "\\.K1\", \"" + dataTableName + ExplainPlanAssert skipScanJoinPlan = assertPlan(conn1, query).scanType("FULL SCAN") + .table(dataTableName).serverWhereFilter("SERVER FILTER BY K3 > 1") + .serverSortedBy("[" + dataTableName + ".V1, " + dataTableName + ".T_ID]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_HINT) + .indexRejectedNone(); + skipScanJoinPlan.subPlanCount(1).subPlan(0) + .abstractExplainPlan("SKIP-SCAN-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(indexTableName).keyRanges("[*] - ['z']") + .serverFirstKeyOnlyProjection(true).indexRule(OptimizerReasons.RULE_DATA_TABLE) + .indexRejectedNone().end(); + // The dynamic server filter references compiler-generated positional aliases ($N.$N) whose + // numbers are not stable, so match the structural shape of the attribute with a regex. + String dynamicServerFilter = skipScanJoinPlan.attributes().getDynamicServerFilter(); + String dynamicServerFilterPattern = "DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + + "\\.T_ID\", \"" + dataTableName + "\\.K1\", \"" + dataTableName + "\\.K2\"\\) IN \\(\\(\\$\\d+\\.\\$\\d+, \\$\\d+\\.\\$\\d+, \\$\\d+\\.\\$\\d+\\)\\)"; - actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, - Pattern.matches(expected, actual)); + assertTrue( + "Expected dynamic server filter to match:\n" + dynamicServerFilterPattern + "\nbut got\n" + + dynamicServerFilter, + dynamicServerFilter != null + && Pattern.matches(dynamicServerFilterPattern, dynamicServerFilter)); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -287,14 +288,11 @@ private void testOptimization(String dataTableName, String dataTableFullName, query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, V1, k3 from " + dataTableFullName + " where v1 <='z' group by v1,t_id, k3"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n" - + " SERVER MERGE [0.K3]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\", \"T_ID\", \"K3\"]\n" - + "CLIENT MERGE SORT"; - - actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); + assertPlan(conn1, query).scanType("RANGE SCAN").table(indexTableName).keyRanges("[*] - ['z']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\", \"T_ID\", \"K3\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_HINT) + .indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -319,12 +317,10 @@ private void testOptimization(String dataTableName, String dataTableFullName, query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ v1,sum(k3) from " + dataTableFullName + " where v1 <='z' group by v1 order by v1"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n" - + " SERVER MERGE [0.K3]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"V1\"]"; - actual = QueryUtil.getExplainPlan(rs); - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); + assertPlan(conn1, query).scanType("RANGE SCAN").table(indexTableName).keyRanges("[*] - ['z']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"V1\"]") + .indexRule(OptimizerReasons.RULE_HINT).indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -356,14 +352,11 @@ private void testOptimizationTenantSpecific(String dataTableName, String indexTa String query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ k1,k2,k3,v1 FROM " + dataTableName + " where v1='a'"; - ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - String actual = QueryUtil.getExplainPlan(rs); - String expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName - + " ['tid1','a']\n" + " SERVER MERGE [0.K3]\n" + " SERVER FILTER BY FIRST KEY ONLY"; - assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected)); + assertPlan(conn1, query).scanType("RANGE SCAN").table(indexTableName) + .keyRanges("['tid1','a']").serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_HINT).indexRejectedNone(); - rs = conn1.createStatement().executeQuery(query); + ResultSet rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(1, rs.getInt("k1")); assertEquals(2, rs.getInt("k2")); @@ -406,19 +399,16 @@ public void testGlobalIndexOptimizationOnSharedIndex() throws Exception { String query = "SELECT /*+ INDEX(" + viewName + " " + viewIndex + ")*/ t_id,k1,k2,k3,v1 FROM " + viewName + " where k1 IN (1,2) and k2 IN (3,4)"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); /** * This inner "_IDX_" + dataTableName use skipScan, and all the whereExpressions are already * in SkipScanFilter, so there is no other RowKeyComparisonFilter needed. */ - String actual = QueryUtil.getExplainPlan(rs); - String expected = "CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER _IDX_" + dataTableName - + " [" + Short.MIN_VALUE + ",1] - [" + Short.MIN_VALUE + ",2]\n" - + " SERVER MERGE [0.K3]\n" + " SERVER FILTER BY FIRST KEY ONLY"; - - assertEquals(expected, actual); + assertPlan(conn1, query).scanType("SKIP SCAN ON 2 KEYS").table("_IDX_" + dataTableName) + .keyRanges("[" + Short.MIN_VALUE + ",1] - [" + Short.MIN_VALUE + ",2]") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_HINT).indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -455,12 +445,11 @@ public void testNoGlobalIndexOptimization() throws Exception { // All columns available in index String query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, k1, k2, V1 FROM " + dataTableName + " where v1='a'"; - ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN " + query); + assertPlan(conn1, query).scanType("RANGE SCAN").table(indexTableName).keyRanges("['a']") + .serverFirstKeyOnlyProjection(true).indexRule(OptimizerReasons.RULE_HINT) + .indexRejectedNone(); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " ['a']\n" - + " SERVER FILTER BY FIRST KEY ONLY", QueryUtil.getExplainPlan(rs)); - - rs = conn1.createStatement().executeQuery(query); + ResultSet rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("f", rs.getString("t_id")); assertEquals(1, rs.getInt("k1")); @@ -473,10 +462,10 @@ public void testNoGlobalIndexOptimization() throws Exception { // No INDEX hint specified query = "SELECT t_id, k1, k2, k3, V1 FROM " + dataTableName + " where v1='a'"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dataTableName + "\n" - + " SERVER FILTER BY V1 = 'a'", QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("FULL SCAN").table(dataTableName) + .serverWhereFilter("SERVER FILTER BY V1 = 'a'") + .indexRule(OptimizerReasons.RULE_ONLY_CANDIDATE).indexRejectedCount(1) + .indexRejected(0, indexTableName, OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -493,12 +482,10 @@ public void testNoGlobalIndexOptimization() throws Exception { // No where clause query = "SELECT t_id, k1, k2, k3, V1 from " + dataTableFullName + " order by V1,t_id"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - assertEquals( - "CLIENT PARALLEL 4-WAY FULL SCAN OVER " + dataTableName + "\n" - + " SERVER SORTED BY [V1, T_ID]\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("FULL SCAN").table(dataTableName) + .serverSortedBy("[V1, T_ID]").clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_ONLY_CANDIDATE).indexRejectedCount(1) + .indexRejected(0, indexTableName, OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -530,11 +517,11 @@ public void testNoGlobalIndexOptimization() throws Exception { // No where clause in index scan query = "SELECT t_id, k1, k2, k3, V1 from " + dataTableFullName + " where k3 > 1 order by V1,t_id"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + dataTableName + "\n" - + " SERVER FILTER BY K3 > 1\n" + " SERVER SORTED BY [V1, T_ID]\n" - + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("FULL SCAN").table(dataTableName) + .serverWhereFilter("SERVER FILTER BY K3 > 1").serverSortedBy("[V1, T_ID]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_ONLY_CANDIDATE) + .indexRejectedCount(1) + .indexRejected(0, indexTableName, OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexWithStatsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexWithStatsIT.java index 575ba7be4d4..d998dd60a70 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexWithStatsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexWithStatsIT.java @@ -17,8 +17,8 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -27,11 +27,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsEnabledIT; import org.apache.phoenix.end2end.ParallelStatsEnabledTest; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.util.PropertiesUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -64,12 +61,7 @@ public void testIndexCreationDeadlockWithStats() throws Exception { conn.createStatement().execute("UPDATE STATISTICS " + tableName); query = "SELECT COUNT(*) FROM " + tableName; - - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN"); String indexName = "I_" + generateUniqueName(); conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + " (v)"); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMaintenanceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMaintenanceIT.java index 0995ced87b5..81f083cc2cf 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMaintenanceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMaintenanceIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end.index; import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; @@ -35,10 +36,10 @@ import org.apache.commons.lang3.StringUtils; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.util.DateUtil; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -157,17 +158,20 @@ private void helpTestCreateAndUpdate(boolean mutable, boolean localIndex) throws stmt.setDate(5, date); // verify that the query does a range scan on the index table - ResultSet rs = stmt.executeQuery("EXPLAIN " + whereSql); - assertEquals(localIndex - ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER INDEX_TEST." + indexName + "(" + fullDataTableName - + ")" - + " [1,'VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]\nCLIENT MERGE SORT" - : "CLIENT PARALLEL 1-WAY RANGE SCAN OVER INDEX_TEST." + indexName - + " ['VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]", - QueryUtil.getExplainPlan(rs)); + if (localIndex) { + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).scanType("RANGE SCAN") + .tableContains("INDEX_TEST." + indexName + "(" + fullDataTableName + ")") + .keyRanges( + "[1,'VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]") + .clientSortAlgo("CLIENT MERGE SORT"); + } else { + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).scanType("RANGE SCAN") + .tableContains("INDEX_TEST." + indexName).keyRanges( + "['VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]"); + } // verify that the correct results are returned - rs = stmt.executeQuery(); + ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); assertEquals(1, rs.getInt(2)); @@ -179,13 +183,14 @@ private void helpTestCreateAndUpdate(boolean mutable, boolean localIndex) throws + "decimal_pk+int_pk+decimal_col2+int_col1, " + "date_pk+1, date1+1, date2+1, " + "varchar_pk, char_pk, int_pk, long_pk, decimal_pk, " + "long_col1, long_col2 " + "from " + fullDataTableName; - rs = conn.createStatement().executeQuery("EXPLAIN " + indexSelectSql); - assertEquals( - localIndex - ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER INDEX_TEST." + indexName + "(" - + fullDataTableName + ") [1]\nCLIENT MERGE SORT" - : "CLIENT PARALLEL 1-WAY FULL SCAN OVER INDEX_TEST." + indexName, - QueryUtil.getExplainPlan(rs)); + if (localIndex) { + assertPlan(conn, indexSelectSql).scanType("RANGE SCAN") + .tableContains("INDEX_TEST." + indexName + "(" + fullDataTableName + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT"); + } else { + assertPlan(conn, indexSelectSql).scanType("FULL SCAN") + .tableContains("INDEX_TEST." + indexName); + } rs = conn.createStatement().executeQuery(indexSelectSql); verifyResult(rs, 1); verifyResult(rs, 2); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexUsageIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexUsageIT.java index c75d1107d79..7b3505cf978 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexUsageIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexUsageIT.java @@ -18,11 +18,11 @@ package org.apache.phoenix.end2end.index; import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -34,17 +34,15 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Properties; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.execute.CommitException; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.explain.ExplainPlanTestUtil; import org.apache.phoenix.util.DateUtil; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -133,24 +131,17 @@ protected void helpTestGroupByCount(boolean mutable, boolean localIndex) throws String groupBySql = "SELECT (int_col1+int_col2), COUNT(*) FROM " + fullDataTableName + " GROUP BY (int_col1+int_col2)"; - ExplainPlan plan = conn.prepareStatement(groupBySql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals( - "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY " - + "[TO_BIGINT(\"(A.INT_COL1 + B.INT_COL2)\")]", - explainPlanAttributes.getServerAggregate()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, groupBySql) + .iteratorType("PARALLEL 1-WAY").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY " + + "[TO_BIGINT(\"(A.INT_COL1 + B.INT_COL2)\")]") + .functionalMatch("(A.INT_COL1 + B.INT_COL2)").indexRejectedNone(); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("INDEX_TEST." + indexName + "(" + fullDataTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN") + .table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("INDEX_TEST." + indexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table("INDEX_TEST." + indexName).clientSortAlgo(null); } ResultSet rs = conn.createStatement().executeQuery(groupBySql); assertTrue(rs.next()); @@ -195,26 +186,18 @@ protected void helpTestSelectDistinct(boolean mutable, boolean localIndex) throw conn.createStatement().execute(ddl); String sql = "SELECT distinct int_col1+1 FROM " + fullDataTableName + " where int_col1+1 > 0"; - ExplainPlan plan = conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("SERVER DISTINCT PREFIX FILTER OVER [TO_BIGINT(\"(A.INT_COL1 + 1)\")]", - explainPlanAttributes.getServerDistinctFilter()); - assertEquals( - "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [TO_BIGINT(\"(A.INT_COL1 + 1)\")]", - explainPlanAttributes.getServerAggregate()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, sql) + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").serverFirstKeyOnlyProjection(true) + .serverDistinctFilter( + "SERVER DISTINCT PREFIX FILTER OVER " + "[TO_BIGINT(\"(A.INT_COL1 + 1)\")]") + .serverAggregate( + "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY " + "[TO_BIGINT(\"(A.INT_COL1 + 1)\")]") + .functionalMatch("(A.INT_COL1 + 1)").indexRejectedNone(); if (localIndex) { - assertEquals("INDEX_TEST." + indexName + "(" + fullDataTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,0] - [1,*]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")") + .keyRanges("[1,0] - [1,*]").clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("INDEX_TEST." + indexName, explainPlanAttributes.getTableName()); - assertEquals(" [0] - [*]", explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.table("INDEX_TEST." + indexName).keyRanges("[0] - [*]").clientSortAlgo(null); } ResultSet rs = conn.createStatement().executeQuery(sql); @@ -262,21 +245,14 @@ protected void helpTestInClauseWithIndex(boolean mutable, boolean localIndex) th conn.createStatement().execute(ddl); String sql = "SELECT int_col1+1 FROM " + fullDataTableName + " where int_col1+1 IN (2)"; - ExplainPlan plan = conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, sql) + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").serverFirstKeyOnlyProjection(true) + .functionalMatch("(A.INT_COL1 + 1)").indexRejectedNone(); if (localIndex) { - assertEquals("INDEX_TEST." + indexName + "(" + fullDataTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,2]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")").keyRanges("[1,2]") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("INDEX_TEST." + indexName, explainPlanAttributes.getTableName()); - assertEquals(" [2]", explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.table("INDEX_TEST." + indexName).keyRanges("[2]").clientSortAlgo(null); } ResultSet rs = conn.createStatement().executeQuery(sql); @@ -323,21 +299,15 @@ protected void helpTestSelectAliasAndOrderByWithIndex(boolean mutable, boolean l conn.createStatement().execute(ddl); String sql = "SELECT int_col1+1 AS foo FROM " + fullDataTableName + " ORDER BY foo"; - ExplainPlan plan = conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY").serverFirstKeyOnlyProjection(true) + .functionalMatch("(A.INT_COL1 + 1)").indexRejectedNone(); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("INDEX_TEST." + indexName + "(" + fullDataTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN") + .table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("INDEX_TEST." + indexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table("INDEX_TEST." + indexName).clientSortAlgo(null); } ResultSet rs = conn.createStatement().executeQuery(sql); @@ -404,19 +374,14 @@ protected void helpTestIndexWithCaseSensitiveCols(boolean mutable, boolean local query = "SELECT (\"V1\" || '_' || \"v2\"), k, \"V1\", \"v2\" FROM " + dataTableName + " WHERE (\"V1\" || '_' || \"v2\") = 'x_1'"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .functionalMatch("(\"cf1\".\"V1\" || '_' || \"CF2\".\"v2\")").indexRejectedNone(); if (localIndex) { - assertEquals(indexName + "(" + dataTableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [1,'x_1']", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.table(indexName + "(" + dataTableName + ")").keyRanges("[1,'x_1']") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals(indexName, explainPlanAttributes.getTableName()); - assertEquals(" ['x_1']", explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.table(indexName).keyRanges("['x_1']").clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -435,19 +400,13 @@ protected void helpTestIndexWithCaseSensitiveCols(boolean mutable, boolean local query = "SELECT \"V1\", \"V1\" as foo1, (\"V1\" || '_' || \"v2\") as foo, (\"V1\" || '_' || \"v2\") as \"Foo1\", (\"V1\" || '_' || \"v2\") FROM " + dataTableName + " ORDER BY foo"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY") + .functionalMatch("(\"cf1\".\"V1\" || '_' || \"CF2\".\"v2\")").indexRejectedNone(); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexName + "(" + dataTableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [1]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("RANGE SCAN").table(indexName + "(" + dataTableName + ")") + .keyRanges("[1]").clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.scanType("FULL SCAN").table(indexName).clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -517,24 +476,18 @@ protected void helpTestSelectColOnlyInDataTable(boolean mutable, boolean localIn conn.createStatement().execute(ddl); String sql = "SELECT int_col1+1, int_col2 FROM " + fullDataTableName + " WHERE int_col1+1=2"; - ExplainPlan plan = conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY"); if (localIndex) { - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("INDEX_TEST." + indexName + "(" + fullDataTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,2]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", - explainPlanAttributes.getServerWhereFilter()); + basePlan.scanType("RANGE SCAN") + .table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")").keyRanges("[1,2]") + .clientSortAlgo("CLIENT MERGE SORT").serverFirstKeyOnlyProjection(true) + .functionalMatch("(A.INT_COL1 + 1)").indexRejectedNone(); } else { - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullDataTableName, explainPlanAttributes.getTableName()); - assertNull(explainPlanAttributes.getClientSortAlgo()); - assertEquals("SERVER FILTER BY (A.INT_COL1 + 1) = 2", - explainPlanAttributes.getServerWhereFilter()); + basePlan.scanType("FULL SCAN").table(fullDataTableName).clientSortAlgo(null) + .serverWhereFilter("SERVER FILTER BY (A.INT_COL1 + 1) = 2") + .indexRule(OptimizerReasons.RULE_ONLY_CANDIDATE).indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION); } ResultSet rs = conn.createStatement().executeQuery(sql); @@ -582,19 +535,15 @@ private void helpTestUpdatableViewIndex(boolean local) throws Exception { conn.commit(); String query = "SELECT k1, k2, k3, s1, s2 FROM " + viewName + " WHERE k1+k2+k3 = 173.0"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .functionalMatch("(K1 + K2 + K3)").indexRejectedNone(); if (local) { - assertEquals(indexName1 + "(" + dataTableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [1,173]", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.table(indexName1 + "(" + dataTableName + ")").keyRanges("[1,173]") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("_IDX_" + dataTableName, explainPlanAttributes.getTableName()); - assertEquals(" [" + Short.MIN_VALUE + ",173]", explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.table("_IDX_" + dataTableName).keyRanges("[" + Short.MIN_VALUE + ",173]") + .clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -610,21 +559,15 @@ private void helpTestUpdatableViewIndex(boolean local) throws Exception { + " on " + viewName + "(s1||'_'||s2)"); query = "SELECT k1, k2, s1||'_'||s2 FROM " + viewName + " WHERE (s1||'_'||s2)='foo2_bar2'"; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .serverFirstKeyOnlyProjection(true).functionalMatch("(S1 || '_' || S2)") + .indexRejectedCount(1); if (local) { - assertEquals(indexName2 + "(" + dataTableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [" + (2) + ",'foo2_bar2']", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.table(indexName2 + "(" + dataTableName + ")") + .keyRanges("[" + (2) + ",'foo2_bar2']").clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals("_IDX_" + dataTableName, explainPlanAttributes.getTableName()); - assertEquals(" [" + (Short.MIN_VALUE + 1) + ",'foo2_bar2']", - explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.table("_IDX_" + dataTableName) + .keyRanges("[" + (Short.MIN_VALUE + 1) + ",'foo2_bar2']").clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -680,14 +623,9 @@ private void helpTestViewUsesTableIndex(boolean immutable) throws Exception { String query = "SELECT s2||'_'||s3 FROM " + viewName + " WHERE k2=1 AND (s2||'_'||s3)='abc_cab'"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(indexName2, explainPlanAttributes.getTableName()); - assertEquals(" [1,'abc_cab','foo']", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(indexName2).keyRanges("[1,'abc_cab','foo']").serverFirstKeyOnlyProjection(true) + .functionalMatch("(S2 || '_' || S3)").indexRejectedCount(1); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -696,11 +634,11 @@ private void helpTestViewUsesTableIndex(boolean immutable) throws Exception { conn.createStatement().execute("ALTER VIEW " + viewName + " DROP COLUMN s4"); // i2 cannot be used since s4 has been dropped from the view, so i1 will be used - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName1 + " [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY AND ((\"S2\" || '_' || \"S3\") = 'abc_cab' AND \"S1\" = 'foo')", - queryPlan); + assertPlan(conn, query).scanType("RANGE SCAN").tableContains(indexName1).keyRanges("[1]") + .serverFirstKeyOnlyProjection(true) + .serverWhereFilter( + "SERVER FILTER BY ((\"S2\" || '_' || \"S3\") = 'abc_cab' AND \"S1\" = 'foo')") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("abc_cab", rs.getString(1)); @@ -784,20 +722,14 @@ protected void helpTestCaseSensitiveFunctionIndex(boolean mutable, boolean local query = "SELECT k FROM " + dataTableName + " WHERE REGEXP_SUBSTR(v,'id:\\\\w+') = 'id:id1'"; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); + ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, query) + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").serverFirstKeyOnlyProjection(true) + .functionalMatch("REGEXP_SUBSTR(V,'id:\\\\w+')").indexRejectedNone(); if (localIndex) { - assertEquals(indexName + "(" + dataTableName + ")", explainPlanAttributes.getTableName()); - assertEquals(" [1,'id:id1']", explainPlanAttributes.getKeyRanges()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + basePlan.table(indexName + "(" + dataTableName + ")").keyRanges("[1,'id:id1']") + .clientSortAlgo("CLIENT MERGE SORT"); } else { - assertEquals(indexName, explainPlanAttributes.getTableName()); - assertEquals(" ['id:id1']", explainPlanAttributes.getKeyRanges()); - assertNull(explainPlanAttributes.getClientSortAlgo()); + basePlan.table(indexName).keyRanges("['id:id1']").clientSortAlgo(null); } rs = conn.createStatement().executeQuery(query); @@ -860,21 +792,22 @@ public void helpTestIndexExpressionWithJoin(boolean mutable, boolean localIndex) query = "select c.c_customer_sk from " + tableName + " c " + "left outer join " + tableName + " c2 on c.c_customer_sk = c2.c_customer_sk " + "where c.c_customer_sk || c.c_first_name = '1David'"; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName + "(" + tableName - + ") [1,'1David']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName + "(" + tableName - + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " CLIENT MERGE SORT", explainPlan); + assertPlan(conn, query).scanType("RANGE SCAN") + .tableContains(indexName + "(" + tableName + ")").keyRanges("[1,'1David']") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT").indexRule(null) + .indexRejectedNone().subPlanCount(1).subPlan(0).scanType("RANGE SCAN") + .tableContains(indexName + "(" + tableName + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .indexRule(null).indexRejectedNone().end(); } else { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName + " ['1David']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " PARALLEL LEFT-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + indexName + "\n" - + " SERVER FILTER BY FIRST KEY ONLY", explainPlan); + assertPlan(conn, query).scanType("RANGE SCAN").tableContains(indexName) + .keyRanges("['1David']").serverFirstKeyOnlyProjection(true).indexRule(null) + .indexRejectedNone().subPlanCount(1).subPlan(0).scanType("FULL SCAN") + .tableContains(indexName).serverFirstKeyOnlyProjection(true) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .indexRule(null).indexRejectedNone().end(); } rs = conn.createStatement().executeQuery(query); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java index ea8f752afde..33409ef4cb5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end.index; import static org.apache.phoenix.end2end.ExplainPlanWithStatsEnabledIT.getByteRowEstimates; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.MetaDataUtil.getViewIndexSequenceName; import static org.apache.phoenix.util.MetaDataUtil.getViewIndexSequenceSchemaName; import static org.junit.Assert.assertArrayEquals; @@ -56,16 +57,14 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Pair; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.end2end.ExplainPlanWithStatsEnabledIT.Estimate; import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; @@ -73,7 +72,6 @@ import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.EnvironmentEdgeManager; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; @@ -438,54 +436,33 @@ public void testUseUncoveredLocalIndexWithPrefix() throws Exception { .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(pk1,pk2,v1,v2)"); // 1. same prefix length, no other restrictions, but v3 is in the SELECT. Use the main table. - ExplainPlan explainPlan = - conn.prepareStatement("SELECT * FROM " + tableName + " WHERE pk1 = 3 AND pk2 = 4") - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(physicalTableName.toString(), explainPlanAttributes.getTableName()); - assertEquals(" [3,4]", explainPlanAttributes.getKeyRanges()); + assertPlan(conn, "SELECT * FROM " + tableName + " WHERE pk1 = 3 AND pk2 = 4") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(physicalTableName.toString()) + .keyRanges("[3,4]").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); // 2. same prefix length, no other restrictions. Only index columns used. Use the index. - explainPlan = - conn.prepareStatement("SELECT v2 FROM " + tableName + " WHERE pk1 = 3 AND pk2 = 4") - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,3,4]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY", explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, "SELECT v2 FROM " + tableName + " WHERE pk1 = 3 AND pk2 = 4") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,3,4]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); // 3. same prefix length, but there's a column not on the index - explainPlan = - conn.prepareStatement("SELECT v2 FROM " + tableName + " WHERE pk1 = 3 AND pk2 = 4 AND v3 = 1") - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(physicalTableName.toString(), explainPlanAttributes.getTableName()); - assertEquals(" [3,4]", explainPlanAttributes.getKeyRanges()); - assertEquals("SERVER FILTER BY V3 = 1", explainPlanAttributes.getServerWhereFilter()); + assertPlan(conn, "SELECT v2 FROM " + tableName + " WHERE pk1 = 3 AND pk2 = 4 AND v3 = 1") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table(physicalTableName.toString()) + .keyRanges("[3,4]").serverWhereFilter("SERVER FILTER BY V3 = 1") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); // 4. Longer prefix on the index, use it. - explainPlan = conn - .prepareStatement( - "SELECT v2 FROM " + tableName + " WHERE pk1 = 3 AND pk2 = 4 AND v1 = 3 AND v3 = 1") - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 1-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals(fullIndexName + "(" + indexPhysicalTableName + ")", - explainPlanAttributes.getTableName()); - assertEquals(" [1,3,4,3]", explainPlanAttributes.getKeyRanges()); - assertEquals("[0.V3]", explainPlanAttributes.getServerMergeColumns().toString()); - assertEquals("SERVER FILTER BY FIRST KEY ONLY AND \"V3\" = 1", - explainPlanAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, + "SELECT v2 FROM " + tableName + " WHERE pk1 = 3 AND pk2 = 4 AND v1 = 3 AND v3 = 1") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,3,4,3]") + .serverMergeColumns("[0.V3]").serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY \"V3\" = 1").clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); } @Test @@ -511,13 +488,12 @@ public void testUseUncoveredLocalIndexWithSplitPrefix() throws Exception { .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(pk1,pk3)"); // 1. Still use the index - ResultSet rs = conn.createStatement().executeQuery( - "EXPLAIN SELECT pk1, pk2, pk3, v1 FROM " + tableName + " WHERE pk1 = 2 AND pk3 = 3"); - assertEquals("CLIENT PARALLEL 16-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1,2,3]\n" + " SERVER MERGE [0.V1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT pk1, pk2, pk3, v1 FROM " + tableName + " WHERE pk1 = 2 AND pk3 = 3") + .iteratorType("PARALLEL").scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,2,3]") + .serverMergeColumns("[0.V1]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); } @Test @@ -541,90 +517,84 @@ public void testUseUncoveredLocalIndex() throws Exception { .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v2, v3, v4)"); // 1. COUNT(*) should still use the index - fewer bytes to scan - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN SELECT COUNT(*) FROM " + tableName); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO SINGLE ROW", QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT COUNT(*) FROM " + tableName).iteratorType("PARALLEL 1-WAY") + .scanType("RANGE SCAN").table(fullIndexName + "(" + indexPhysicalTableName + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO SINGLE ROW") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); // 2. All column projected, no filtering by indexed column, not using the index - rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + tableName); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + physicalTableName, - QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName).iteratorType("PARALLEL 1-WAY") + .scanType("FULL SCAN").table(physicalTableName.toString()) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); // 3. if the index can avoid a sort operation, use it - rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + tableName + " ORDER BY v2"); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1]\n" + " SERVER MERGE [0.V1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " ORDER BY v2").iteratorType("PARALLEL 1-WAY") + .scanType("RANGE SCAN").table(fullIndexName + "(" + indexPhysicalTableName + ")") + .keyRanges("[1]").serverMergeColumns("[0.V1]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); // 4. but can't use the index if not ORDERing by a prefix of the index key. - rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + tableName + " ORDER BY v3"); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + physicalTableName + "\n" - + " SERVER SORTED BY [V3]\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " ORDER BY v3").iteratorType("PARALLEL 1-WAY") + .scanType("FULL SCAN").table(physicalTableName.toString()).serverSortedBy("[V3]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); // 5. If we pin the prefix of the index key we use the index avoiding sorting on the postfix - rs = conn.createStatement() - .executeQuery("EXPLAIN SELECT * FROM " + tableName + " WHERE v2 = 2 ORDER BY v3"); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1,2]\n" + " SERVER MERGE [0.V1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " WHERE v2 = 2 ORDER BY v3") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,2]") + .serverMergeColumns("[0.V1]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); // 6. Filtering by a non-indexed column will not use the index - rs = - conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + tableName + " WHERE v1 = 3"); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + physicalTableName + "\n" - + " SERVER FILTER BY V1 = 3.0", QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " WHERE v1 = 3").iteratorType("PARALLEL 1-WAY") + .scanType("FULL SCAN").table(physicalTableName.toString()) + .serverWhereFilter("SERVER FILTER BY V1 = 3.0") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); // 7. Also don't use an index if not filtering on a prefix of the key - rs = - conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + tableName + " WHERE v3 = 1"); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + physicalTableName + "\n" - + " SERVER FILTER BY V3 = 1", QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " WHERE v3 = 1").iteratorType("PARALLEL 1-WAY") + .scanType("FULL SCAN").table(physicalTableName.toString()) + .serverWhereFilter("SERVER FILTER BY V3 = 1") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); // 8. Filtering along a prefix of the index key can use the index - rs = - conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + tableName + " WHERE v2 = 2"); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1,2]\n" + " SERVER MERGE [0.V1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " WHERE v2 = 2").iteratorType("PARALLEL 1-WAY") + .scanType("RANGE SCAN").table(fullIndexName + "(" + indexPhysicalTableName + ")") + .keyRanges("[1,2]").serverMergeColumns("[0.V1]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); // 9. Make sure a gap in the index columns still uses the index as long as a prefix is specified - rs = conn.createStatement() - .executeQuery("EXPLAIN SELECT * FROM " + tableName + " WHERE v2 = 2 AND v4 = 4"); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + indexPhysicalTableName - + ") [1,2]\n" + " SERVER MERGE [0.V1]\n" - + " SERVER FILTER BY FIRST KEY ONLY AND TO_INTEGER(\"V4\") = 4\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " WHERE v2 = 2 AND v4 = 4") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,2]") + .serverMergeColumns("[0.V1]").serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY TO_INTEGER(\"V4\") = 4") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); // 10. Use index even when also filtering on non-indexed column - rs = conn.createStatement() - .executeQuery("EXPLAIN SELECT * FROM " + tableName + " WHERE v2 = 2 AND v1 = 3"); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + indexPhysicalTableName - + ") [1,2]\n" + " SERVER MERGE [0.V1]\n" - + " SERVER FILTER BY FIRST KEY ONLY AND \"V1\" = 3.0\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " WHERE v2 = 2 AND v1 = 3") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,2]") + .serverMergeColumns("[0.V1]").serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY \"V1\" = 3.0").clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); // 11. Another case of not using a prefix of the index key - rs = conn.createStatement() - .executeQuery("EXPLAIN SELECT * FROM " + tableName + " WHERE v1 = 3 AND v3 = 1 AND v4 = 1"); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + physicalTableName + "\n" - + " SERVER FILTER BY (V1 = 3.0 AND V3 = 1 AND V4 = 1)", QueryUtil.getExplainPlan(rs)); - rs.close(); + assertPlan(conn, "SELECT * FROM " + tableName + " WHERE v1 = 3 AND v3 = 1 AND v4 = 1") + .iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN").table(physicalTableName.toString()) + .serverWhereFilter("SERVER FILTER BY (V1 = 3.0 AND V3 = 1 AND V4 = 1)") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); } @Test @@ -829,19 +799,13 @@ public void testLocalIndexUsedForUncoveredOrderBy() throws Exception { .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)"); String query = "SELECT * FROM " + tableName + " ORDER BY V1"; - ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - Admin admin = - driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - int numRegions = admin.getRegions(physicalTableName).size(); - - assertEquals( - "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1]\n" + " SERVER MERGE [0.K3]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1]") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); - rs = conn1.createStatement().executeQuery(query); + ResultSet rs = conn1.createStatement().executeQuery(query); String v = ""; int i = 0; while (rs.next()) { @@ -855,12 +819,11 @@ public void testLocalIndexUsedForUncoveredOrderBy() throws Exception { rs.close(); query = "SELECT * FROM " + tableName + " ORDER BY V1 DESC NULLS LAST"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - assertEquals( - "CLIENT PARALLEL " + numRegions + "-WAY REVERSE RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1]\n" + " SERVER MERGE [0.K3]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("RANGE SCAN").clientSortedBy("REVERSE") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1]") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); v = "zz"; @@ -898,17 +861,12 @@ public void testLocalIndexReverseScanShouldReturnAllRows() throws Exception { .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)"); String query = "SELECT V1 FROM " + tableName + " ORDER BY V1 DESC NULLS LAST"; - ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - Admin admin = - driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - int numRegions = admin.getRegions(physicalTableName).size(); + assertPlan(conn1, query).scanType("RANGE SCAN").clientSortedBy("REVERSE") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); - assertEquals("CLIENT PARALLEL " + numRegions + "-WAY REVERSE RANGE SCAN OVER " + fullIndexName - + "(" + indexPhysicalTableName + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); - - rs = conn1.createStatement().executeQuery(query); + ResultSet rs = conn1.createStatement().executeQuery(query); String v = "zz"; int i = 0; while (rs.next()) { @@ -948,18 +906,12 @@ public void testLocalIndexScanJoinColumnsFromDataTable() throws Exception { ResultSet rs = conn1.createStatement().executeQuery("SELECT COUNT(*) FROM " + indexTableName); assertTrue(rs.next()); - Admin admin = - driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); - int numRegions = admin.getRegions(physicalTableName).size(); - String query = "SELECT t_id, k1, k2, k3, V1 FROM " + tableName + " where v1='a'"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - assertEquals( - "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1,'a']\n" + " SERVER MERGE [0.K3]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,'a']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -975,13 +927,11 @@ public void testLocalIndexScanJoinColumnsFromDataTable() throws Exception { assertFalse(rs.next()); query = "SELECT t_id, k1, k2, k3, V1 from " + tableName + " where v1<='z' order by V1,t_id"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - assertEquals( - "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1,*] - [1,'z']\n" + " SERVER MERGE [0.K3]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,*] - [1,'z']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -1010,13 +960,12 @@ public void testLocalIndexScanJoinColumnsFromDataTable() throws Exception { assertEquals("z", rs.getString("V1")); query = "SELECT t_id, V1, k3 from " + tableName + " where v1 <='z' group by v1,t_id, k3"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - - assertEquals("CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1,*] - [1,'z']\n" + " SERVER MERGE [0.K3]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\", \"T_ID\", \"K3\"]\nCLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,*] - [1,'z']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\", \"T_ID\", \"K3\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); rs = conn1.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -1038,13 +987,12 @@ public void testLocalIndexScanJoinColumnsFromDataTable() throws Exception { query = "SELECT v1,sum(k3) from " + tableName + " where v1 <='z' group by v1 order by v1"; - rs = conn1.createStatement().executeQuery("EXPLAIN " + query); - assertEquals( - "CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + fullIndexName + "(" - + indexPhysicalTableName + ") [1,*] - [1,'z']\n" + " SERVER MERGE [0.K3]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"V1\"]\nCLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn1, query).scanType("RANGE SCAN") + .table(fullIndexName + "(" + indexPhysicalTableName + ")").keyRanges("[1,*] - [1,'z']") + .serverMergeColumns("[0.K3]").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"V1\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); PhoenixStatement stmt = conn1.createStatement().unwrap(PhoenixStatement.class); rs = stmt.executeQuery(query); @@ -1086,10 +1034,12 @@ public void testIndexPlanSelectionIfBothGlobalAndLocalIndexesHasSameColumnsAndOr conn1.createStatement() .execute("CREATE INDEX " + indexName + "2" + " ON " + tableName + "(v1)"); String query = "SELECT t_id, k1, k2,V1 FROM " + tableName + " where v1='a'"; - ResultSet rs1 = conn1.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + SchemaUtil.getPhysicalTableName(Bytes.toBytes(indexTableName), isNamespaceMapped) + "2" - + " ['a']\n" + " SERVER FILTER BY FIRST KEY ONLY", QueryUtil.getExplainPlan(rs1)); + assertPlan(conn1, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table( + SchemaUtil.getPhysicalTableName(Bytes.toBytes(indexTableName), isNamespaceMapped) + "2") + .keyRanges("['a']").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedCount(1) + .indexRejected(0, indexName, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); conn1.close(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java index 13a514f3304..3577e7435d4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java @@ -18,6 +18,7 @@ package org.apache.phoenix.end2end.index; import static org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -49,8 +50,6 @@ import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.coprocessor.MetaDataRegionObserver; import org.apache.phoenix.coprocessor.MetaDataRegionObserver.BuildIndexScheduleTask; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; @@ -60,7 +59,6 @@ import org.apache.phoenix.index.PhoenixIndexFailurePolicy; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; @@ -303,15 +301,10 @@ public void testIndexWriteFailure() throws Exception { addRowToTable(conn, fullTableName); query = "SELECT /*+ NO_INDEX */ k,v1 FROM " + fullTableName; - ExplainPlan plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class) - .optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 2-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals( - SchemaUtil.getPhysicalTableName(fullTableName.getBytes(), isNamespaceMapped).toString(), - explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, query).iteratorType("PARALLEL 2-WAY").scanType("FULL SCAN") + .table( + SchemaUtil.getPhysicalTableName(fullTableName.getBytes(), isNamespaceMapped).toString()) + .clientSortAlgo("CLIENT MERGE SORT"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -353,15 +346,10 @@ public void testIndexWriteFailure() throws Exception { updateTableAgain(conn, false); // Verify previous writes succeeded to data table query = "SELECT /*+ NO_INDEX */ k,v1 FROM " + fullTableName; - plan = conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class).optimizeQuery() - .getExplainPlan(); - explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals("PARALLEL 2-WAY", explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); - assertEquals( - SchemaUtil.getPhysicalTableName(fullTableName.getBytes(), isNamespaceMapped).toString(), - explainPlanAttributes.getTableName()); - assertEquals("CLIENT MERGE SORT", explainPlanAttributes.getClientSortAlgo()); + assertPlan(conn, query).iteratorType("PARALLEL 2-WAY").scanType("FULL SCAN") + .table( + SchemaUtil.getPhysicalTableName(fullTableName.getBytes(), isNamespaceMapped).toString()) + .clientSortAlgo("CLIENT MERGE SORT"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -458,17 +446,14 @@ private void addRowsInTableDuringRetry(final String tableName) boolean wasFailWrite = FailingRegionObserver.FAIL_WRITE; boolean wasToggleFailWriteForRetry = FailingRegionObserver.TOGGLE_FAIL_WRITE_FOR_RETRY; try { - Callable callable = new Callable() { - + Callable callable = new Callable() { @Override public Boolean call() { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, String.valueOf(isNamespaceMapped)); try (Connection conn = driver.connect(url, props)) { // In case of disable index on failure policy, INDEX will be in PENDING_DISABLE on first - // retry - // but will - // become active if retry is successfull + // retry but will become active if retry is successfull PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?)"); stmt.setString(1, "b"); @@ -511,18 +496,15 @@ private void validateDataWithIndex(Connection conn, String fullTableName, String String query = "SELECT /*+ INDEX(" + fullTableName + " " + SchemaUtil.getTableNameFromFullName(fullIndexName) + ") */ k,v1 FROM " + fullTableName; ResultSet rs = conn.createStatement().executeQuery(query); - String expectedPlan = - " OVER " - + (localIndex - ? fullIndexName + "(" - + Bytes.toString(SchemaUtil - .getPhysicalTableName(fullTableName.getBytes(), isNamespaceMapped).getName()) - + ")" - : SchemaUtil.getPhysicalTableName(fullIndexName.getBytes(), isNamespaceMapped) - .getNameAsString()); - String explainPlan = - QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN " + query)); - assertTrue(explainPlan, explainPlan.contains(expectedPlan)); + String expectedTable = localIndex + ? fullIndexName + "(" + + Bytes.toString( + SchemaUtil.getPhysicalTableName(fullTableName.getBytes(), isNamespaceMapped).getName()) + + ")" + : SchemaUtil.getPhysicalTableName(fullIndexName.getBytes(), isNamespaceMapped) + .getNameAsString(); + assertPlan(conn, query).scanType(localIndex ? "RANGE SCAN" : "FULL SCAN") + .tableContains(expectedTable); if (transactional) { // failed commit does not get retried assertTrue(rs.next()); assertEquals("a", rs.getString(1)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java index 116cc574db8..9d648f082ab 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -38,13 +39,13 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.util.IndexScrutiny; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; @@ -115,16 +116,18 @@ public void testCoveredColumnUpdates() throws Exception { + fullTableName + " (char_col1 ASC, int_col1 ASC) INCLUDE (long_col1, long_col2)"); String query = "SELECT char_col1, int_col1, long_col2 from " + fullTableName; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + fullTableName - + ") [1]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); } else { - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName, - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(fullIndexName).indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); } - rs = conn.createStatement().executeQuery(query); + ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("chara", rs.getString(1)); assertEquals(2, rs.getInt(2)); @@ -185,11 +188,12 @@ public void testCoveredColumnUpdates() throws Exception { assertFalse(rs.next()); if (localIndex) { query = "SELECT b.* from " + fullTableName + " where int_col1 = 4 AND char_col1 = 'chara'"; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + fullTableName - + ") [1,'chara',4]\n" - + " SERVER MERGE [B.VARCHAR_COL2, B.CHAR_COL2, B.INT_COL2, B.DECIMAL_COL2, B.DATE_COL]\n" - + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1,'chara',4]") + .serverMergeColumns( + "[B.VARCHAR_COL2, B.CHAR_COL2, B.INT_COL2, B.DECIMAL_COL2, B.DATE_COL]") + .clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("varchar_b", rs.getString(1)); @@ -286,13 +290,15 @@ public void testCoveredColumns() throws Exception { assertFalse(rs.next()); query = "SELECT * FROM " + fullTableName; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + fullTableName - + ") [1]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); } else { - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName, - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(fullIndexName).indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); } rs = conn.createStatement().executeQuery(query); @@ -309,13 +315,15 @@ public void testCoveredColumns() throws Exception { conn.commit(); query = "SELECT * FROM " + fullTableName; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + fullTableName - + ") [1]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); } else { - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName, - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(fullIndexName).indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); } rs = conn.createStatement().executeQuery(query); @@ -332,13 +340,15 @@ public void testCoveredColumns() throws Exception { conn.commit(); query = "SELECT * FROM " + fullTableName; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + fullTableName - + ") [1]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); } else { - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName, - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(fullIndexName).indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedNone(); } rs = conn.createStatement().executeQuery(query); @@ -405,16 +415,15 @@ public void testCompoundIndexKey() throws Exception { assertFalse(rs.next()); query = "SELECT * FROM " + fullTableName; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + fullTableName - + ") [1]\n" + " SERVER FILTER BY " + (columnEncoded ? "FIRST KEY" : "EMPTY COLUMN") - + " ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1]") + .serverProjectionFilter(columnEncoded).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); } else { - assertEquals( - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName + "\n" + " SERVER FILTER BY " - + (columnEncoded ? "FIRST KEY" : "EMPTY COLUMN") + " ONLY", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(fullIndexName).serverProjectionFilter(columnEncoded) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); } // make sure the data table looks like what we expect rs = conn.createStatement().executeQuery(query); @@ -534,16 +543,15 @@ public void testMultipleUpdatesToSingleRow() throws Exception { assertFalse(rs.next()); query = "SELECT * FROM " + fullTableName; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" + fullTableName - + ") [1]\n" + " SERVER FILTER BY " + (columnEncoded ? "FIRST KEY" : "EMPTY COLUMN") - + " ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + fullTableName + ")").keyRanges("[1]") + .serverProjectionFilter(columnEncoded).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); } else { - assertEquals( - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + fullIndexName + "\n" + " SERVER FILTER BY " - + (columnEncoded ? "FIRST KEY" : "EMPTY COLUMN") + " ONLY", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(fullIndexName).serverProjectionFilter(columnEncoded) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); } // check that the data table matches as expected diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexIT.java index aef493217b8..f59c9b74e25 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexIT.java @@ -47,8 +47,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.mapreduce.CounterGroup; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.IndexToolIT; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.exception.PhoenixParserException; @@ -57,14 +55,15 @@ import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.mapreduce.index.IndexTool; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.explain.ExplainPlanTestUtil; import org.apache.phoenix.schema.ColumnNotFoundException; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.After; import org.junit.Assert; @@ -110,7 +109,7 @@ public static synchronized Collection data() { { false, true, true } }); } - public static void assertPlan(PhoenixResultSet rs, String schemaName, String tableName) { + public static void assertCurrentTable(PhoenixResultSet rs, String schemaName, String tableName) { PTable table = rs.getContext().getCurrentTable().getTable(); assertTrue(table.getSchemaName().getString().equals(schemaName) && table.getTableName().getString().equals(tableName)); @@ -239,23 +238,21 @@ public void testAtomicUpsert() throws Exception { String selectSql = "SELECT D from " + dataTableName + " WHERE A > 60"; ResultSet rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is used - assertPlan((PhoenixResultSet) rs, "", indexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", indexTableName); assertTrue(rs.next()); assertEquals("b", rs.getString(1)); assertFalse(rs.next()); // explain plan verify to check if partial index is used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(indexTableName)); + assertTrue( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).toString().contains(indexTableName)); selectSql = "SELECT D from " + dataTableName + " WHERE A = 50"; rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is not used - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); // explain plan verify to check if partial index is not used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertFalse(rs.getString(1).contains(indexTableName)); + assertFalse( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).get(0).contains(indexTableName)); // Add more rows to test the index write path conn.createStatement() @@ -289,13 +286,12 @@ public void testAtomicUpsert() throws Exception { // Retrieve update row from the data table and verify that the index table is not used selectSql = "SELECT ID from " + dataTableName + " WHERE A = 0"; rs = conn.createStatement().executeQuery(selectSql); - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); assertTrue(rs.next()); assertEquals("id2", rs.getString(1)); // explain plan verify to check if partial index is not used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertFalse(rs.getString(1).contains(indexTableName)); + assertFalse( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).get(0).contains(indexTableName)); // Test index verification and repair by IndexTool verifyIndex(dataTableName, indexTableName); @@ -307,6 +303,36 @@ public void testAtomicUpsert() throws Exception { } } + @Test + public void testPartialIndexBreadcrumbsAreDistinctPerIndex() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + String dataTableName = generateUniqueName(); + conn.createStatement() + .execute("create table " + dataTableName + " (id varchar not null primary key, " + + "A integer, B integer, C double, D varchar) COLUMN_ENCODED_BYTES=0" + + (salted ? ", SALT_BUCKETS=4" : "")); + // Two partial indexes on the same table with different WHERE predicates. The index is + // created on an empty table so it is active immediately without an async rebuild. + String indexName1 = generateUniqueName(); + String indexName2 = generateUniqueName(); + conn.createStatement() + .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + (local ? "LOCAL " : " ") + "INDEX " + + indexName1 + " on " + dataTableName + " (A) " + (uncovered ? "" : "INCLUDE (B, C, D)") + + " WHERE A > 50"); + conn.createStatement() + .execute("CREATE " + (uncovered ? "UNCOVERED " : " ") + (local ? "LOCAL " : " ") + "INDEX " + + indexName2 + " on " + dataTableName + " (B) " + (uncovered ? "" : "INCLUDE (A, C, D)") + + " WHERE B > 50"); + // A query whose WHERE implies both index WHERE clauses makes the optimizer evaluate both + // partial indexes, so both applicability decisions are recorded. Each breadcrumb names its + // index. + String selectSql = "SELECT D from " + dataTableName + " WHERE A > 60 AND B > 60"; + ExplainPlanTestUtil.assertPlan(conn, selectSql) + .rewriteContains("PARTIAL INDEX " + indexName1 + " APPLICABLE") + .rewriteContains("PARTIAL INDEX " + indexName2 + " APPLICABLE"); + } + } + @Test public void testComparisonOfColumns() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl())) { @@ -336,23 +362,21 @@ public void testComparisonOfColumns() throws Exception { ResultSet rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is used - assertPlan((PhoenixResultSet) rs, "", indexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", indexTableName); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); assertFalse(rs.next()); // explain plan verify to check if partial index is used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(indexTableName)); + assertTrue( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).toString().contains(indexTableName)); selectSql = "SELECT D from " + dataTableName + " WHERE A > 100"; rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is not used - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); // explain plan verify to check if partial index is not used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertFalse(rs.getString(1).contains(indexTableName)); + assertFalse( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).get(0).contains(indexTableName)); // Add more rows to test the index write path conn.createStatement() @@ -424,15 +448,14 @@ public void testIsNull() throws Exception { ResultSet rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is used - assertPlan((PhoenixResultSet) rs, "", indexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", indexTableName); assertTrue(rs.next()); assertEquals(70, rs.getInt(1)); assertEquals("a", rs.getString(2)); assertFalse(rs.next()); // explain plan verify to check if partial index is used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(indexTableName)); + assertTrue( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).toString().contains(indexTableName)); // Add more rows to test the index write path conn.createStatement() @@ -457,14 +480,12 @@ public void testIsNull() throws Exception { rs = conn.createStatement().executeQuery("SELECT Count(*) from " + dataTableName); // Verify that the index table is not used - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); assertTrue(rs.next()); assertEquals(5, rs.getInt(1)); // explain plan verify to check if partial index is not used - rs = - conn.createStatement().executeQuery("EXPLAIN " + "SELECT Count(*) from " + dataTableName); - assertTrue(rs.next()); - assertFalse(rs.getString(1).contains(indexTableName)); + assertFalse(ExplainPlanTestUtil.getPlanSteps(conn, "SELECT Count(*) from " + dataTableName) + .get(0).contains(indexTableName)); // Overwrite an existing row that satisfies the index WHERE clause such that // the new version of the row does not satisfy the index where clause anymore. This @@ -515,14 +536,13 @@ public void testLike() throws Exception { "SELECT D from " + dataTableName + " WHERE B is not NULL AND D like '%cde_'"; ResultSet rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is used - assertPlan((PhoenixResultSet) rs, "", indexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", indexTableName); assertTrue(rs.next()); assertEquals("abcdef", rs.getString(1)); assertFalse(rs.next()); // explain plan verify to check if partial index is used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(indexTableName)); + assertTrue( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).toString().contains(indexTableName)); // Add more rows to test the index write path conn.createStatement() @@ -545,13 +565,12 @@ public void testLike() throws Exception { selectSql = "SELECT Count(*) from " + dataTableName; rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is not used - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); assertTrue(rs.next()); assertEquals(5, rs.getInt(1)); // explain plan verify to check if partial index is not used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertFalse(rs.getString(1).contains(indexTableName)); + assertFalse( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).get(0).contains(indexTableName)); // Overwrite an existing row that satisfies the index WHERE clause such that // the new version of the row does not satisfy the index where clause anymore. This @@ -605,16 +624,15 @@ public void testPhoenixRowTimestamp() throws Exception { + "', 'yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; ResultSet rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is used - assertPlan((PhoenixResultSet) rs, "", indexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", indexTableName); assertTrue(rs.next()); assertEquals(0, rs.getInt(1)); assertTrue(rs.next()); assertEquals(70, rs.getInt(1)); assertFalse(rs.next()); // explain plan verify to check if partial index is used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(indexTableName)); + assertTrue( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).toString().contains(indexTableName)); // Add more rows to test the index write path conn.createStatement().execute("upsert into " + dataTableName + " values ('id2', 20, 3)"); @@ -624,14 +642,12 @@ public void testPhoenixRowTimestamp() throws Exception { rs = conn.createStatement().executeQuery("SELECT Count(*) from " + dataTableName); // Verify that the index table is not used - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); assertTrue(rs.next()); assertEquals(4, rs.getInt(1)); // explain plan verify to check if partial index is not used - rs = - conn.createStatement().executeQuery("EXPLAIN " + "SELECT Count(*) from " + dataTableName); - assertTrue(rs.next()); - assertFalse(rs.getString(1).contains(indexTableName)); + assertFalse(ExplainPlanTestUtil.getPlanSteps(conn, "SELECT Count(*) from " + dataTableName) + .get(0).contains(indexTableName)); rs = conn.createStatement().executeQuery("SELECT Count(*) from " + indexTableName); assertTrue(rs.next()); @@ -645,14 +661,13 @@ public void testPhoenixRowTimestamp() throws Exception { rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is used - assertPlan((PhoenixResultSet) rs, "", indexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", indexTableName); assertTrue(rs.next()); assertEquals(0, rs.getInt(1)); assertFalse(rs.next()); // explain plan verify to check if partial index is used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(indexTableName)); + assertTrue( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).toString().contains(indexTableName)); // Test index verification and repair by IndexTool verifyIndex(dataTableName, indexTableName); @@ -723,7 +738,7 @@ public void testViewIndexes() throws Exception { // Verify that query uses the tenant view index ResultSet rs = tenantConn.createStatement() .executeQuery("SELECT KV1 FROM " + tenantViewName + " WHERE PK3 = 5"); - assertPlan((PhoenixResultSet) rs, "", tenantViewIndexName); + assertCurrentTable((PhoenixResultSet) rs, "", tenantViewIndexName); assertTrue(rs.next()); assertEquals("KV10", rs.getString(1)); assertFalse(rs.next()); @@ -732,7 +747,7 @@ public void testViewIndexes() throws Exception { // where clause does not contain the query where clause rs = tenantConn.createStatement() .executeQuery("SELECT KV1 FROM " + tenantViewName + " WHERE PK3 = 4"); - assertPlan((PhoenixResultSet) rs, "", tenantViewName); + assertCurrentTable((PhoenixResultSet) rs, "", tenantViewName); assertTrue(rs.next()); assertEquals("KV8", rs.getString(1)); assertFalse(rs.next()); @@ -740,7 +755,7 @@ public void testViewIndexes() throws Exception { // Verify that the tenant view index has only one row rs = tenantConn.createStatement().executeQuery("SELECT Count(*) FROM " + tenantViewIndexName); - assertPlan((PhoenixResultSet) rs, "", tenantViewIndexName); + assertCurrentTable((PhoenixResultSet) rs, "", tenantViewIndexName); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); } @@ -751,7 +766,7 @@ public void testViewIndexes() throws Exception { // Verify that the query uses the global view index ResultSet rs = tenantConn.createStatement() .executeQuery("SELECT KV1 FROM " + tenantViewName + " WHERE PK3 = 1 AND KV3 = 'KV3'"); - assertPlan((PhoenixResultSet) rs, "", tenantViewName + "#" + globalViewIndexName); + assertCurrentTable((PhoenixResultSet) rs, "", tenantViewName + "#" + globalViewIndexName); assertTrue(rs.next()); assertEquals("KV1", rs.getString(1)); assertFalse(rs.next()); @@ -760,14 +775,14 @@ public void testViewIndexes() throws Exception { // Verify that the query uses the global view index ResultSet rs = conn.createStatement() .executeQuery("SELECT KV1 FROM " + globalViewName + " WHERE PK3 = 1 AND KV3 = 'KV3'"); - assertPlan((PhoenixResultSet) rs, "", globalViewIndexName); + assertCurrentTable((PhoenixResultSet) rs, "", globalViewIndexName); assertTrue(rs.next()); assertEquals("KV1", rs.getString(1)); assertFalse(rs.next()); // Verify that the global view index has five rows rs = conn.createStatement().executeQuery("SELECT Count(*) FROM " + globalViewIndexName); - assertPlan((PhoenixResultSet) rs, "", globalViewIndexName); + assertCurrentTable((PhoenixResultSet) rs, "", globalViewIndexName); assertTrue(rs.next()); assertEquals(5, rs.getInt(1)); } @@ -801,26 +816,24 @@ public void testPartialIndexPreferredOverFullIndex() throws Exception { String selectSql = "SELECT D from " + dataTableName + " WHERE A > 60"; // Verify that the partial index table is used ResultSet rs = conn.createStatement().executeQuery(selectSql); - assertPlan((PhoenixResultSet) rs, "", partialIndexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", partialIndexTableName); assertTrue(rs.next()); assertEquals("b", rs.getString(1)); assertFalse(rs.next()); // explain plan verify to check if partial index is used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(partialIndexTableName)); + assertTrue(ExplainPlanTestUtil.getPlanSteps(conn, selectSql).toString() + .contains(partialIndexTableName)); selectSql = "SELECT D from " + dataTableName + " WHERE A < 50"; // Verify that the full index table is used rs = conn.createStatement().executeQuery(selectSql); - assertPlan((PhoenixResultSet) rs, "", fullIndexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", fullIndexTableName); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); assertFalse(rs.next()); // explain plan verify to check if full index is used - rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - assertTrue(rs.next()); - assertTrue(rs.getString(1).contains(fullIndexTableName)); + assertTrue( + ExplainPlanTestUtil.getPlanSteps(conn, selectSql).toString().contains(fullIndexTableName)); } } @@ -855,7 +868,7 @@ public void testPartialIndexWithJson() throws Exception { + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) > 60"; ResultSet rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is used - assertPlan((PhoenixResultSet) rs, "", indexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", indexTableName); assertTrue(rs.next()); assertEquals("b", rs.getString(1)); assertFalse(rs.next()); @@ -864,7 +877,7 @@ public void testPartialIndexWithJson() throws Exception { + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 50"; rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is not used - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); // Add more rows to test the index write path conn.createStatement().execute("upsert into " + dataTableName @@ -896,7 +909,7 @@ public void testPartialIndexWithJson() throws Exception { selectSql = "SELECT ID from " + dataTableName + " WHERE (CAST(TO_NUMBER(JSON_VALUE(jsoncol, '$.info.age')) AS INTEGER)) = 0"; rs = conn.createStatement().executeQuery(selectSql); - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); assertTrue(rs.next()); assertEquals("id2", rs.getString(1)); @@ -943,7 +956,7 @@ public void testPartialIndexWithJsonExists() throws Exception { + " WHERE A > 60 AND JSON_EXISTS(jsoncol, '$.info.address.exists')"; ResultSet rs = conn.createStatement().executeQuery(selectSql); // Verify that the index table is used - assertPlan((PhoenixResultSet) rs, "", indexTableName); + assertCurrentTable((PhoenixResultSet) rs, "", indexTableName); assertTrue(rs.next()); assertEquals(70, rs.getInt(1)); assertEquals("a", rs.getString(2)); @@ -969,7 +982,7 @@ public void testPartialIndexWithJsonExists() throws Exception { rs = conn.createStatement().executeQuery("SELECT Count(*) from " + dataTableName); // Verify that the index table is not used - assertPlan((PhoenixResultSet) rs, "", dataTableName); + assertCurrentTable((PhoenixResultSet) rs, "", dataTableName); assertTrue(rs.next()); assertEquals(5, rs.getInt(1)); @@ -1014,26 +1027,23 @@ public void testPartialIndexWithIndexHint() throws Exception { // Index hint provided and query plan using partial index is usable String selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ " + "A from " + dataTableName + " WHERE id2 = 100 AND id1 = 'id12'"; - ResultSet rs = stmt.executeQuery("EXPLAIN " + selectSql); - String actualQueryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualQueryPlan.contains("POINT LOOKUP ON 1 KEY OVER " + indexTableName)); - rs = stmt.executeQuery(selectSql); + ExplainPlanTestUtil.assertPlan(conn, selectSql).scanType("POINT LOOKUP ON 1 KEY") + .table(indexTableName).indexRule(OptimizerReasons.RULE_HINT).indexRejectedNone(); + ResultSet rs = stmt.executeQuery(selectSql); assertTrue(rs.next()); assertEquals(2, rs.getInt(1)); assertFalse(rs.next()); // Index hint provided but query plan using partial index is not usable so, no data selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ " + "A from " + dataTableName + " WHERE id2 = 10 AND id1 = 'id11'"; - rs = stmt.executeQuery("EXPLAIN " + selectSql); - actualQueryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualQueryPlan.contains("POINT LOOKUP ON 1 KEY OVER " + indexTableName)); + ExplainPlanTestUtil.assertPlan(conn, selectSql).scanType("POINT LOOKUP ON 1 KEY") + .table(indexTableName).indexRule(null).indexRejectedNone(); rs = stmt.executeQuery(selectSql); assertFalse(rs.next()); // No index hint so, use data table only as its point lookup selectSql = "SELECT A from " + dataTableName + " WHERE id2 = 10 AND id1 = 'id11'"; - rs = stmt.executeQuery("EXPLAIN " + selectSql); - actualQueryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualQueryPlan.contains("POINT LOOKUP ON 1 KEY OVER " + dataTableName)); + ExplainPlanTestUtil.assertPlan(conn, selectSql).scanType("POINT LOOKUP ON 1 KEY") + .table(dataTableName).indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); rs = stmt.executeQuery(selectSql); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); @@ -1192,13 +1202,9 @@ public void testPartialIndexWithVarbinaryEncoded() throws Exception { private static void verifyIndexUsed(PreparedStatement preparedStatement, String partialIndexTableName, int buckets) throws SQLException { - ExplainPlan plan = - preparedStatement.unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes explainPlanAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(partialIndexTableName, explainPlanAttributes.getTableName()); - assertEquals("PARALLEL " + buckets + "-WAY", - explainPlanAttributes.getIteratorTypeAndScanSize()); - assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType()); + ExplainPlanTestUtil.assertPlan(preparedStatement.unwrap(PhoenixPreparedStatement.class)) + .table(partialIndexTableName).iteratorType("PARALLEL " + buckets + "-WAY") + .scanType("RANGE SCAN"); } private static String toStringLiteral(byte[] b) { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java index 8184f923962..19c8b23655f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -22,6 +22,7 @@ import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.COLUMN_TYPES; import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_COLUMNS; import static org.apache.phoenix.query.QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -31,16 +32,12 @@ import java.sql.Connection; import java.sql.DriverManager; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -168,7 +165,7 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT { // SQL on the index table - SYSTEM.SYS_VIEW_INDEX_HDR_TEST_INDEX, static final String SYS_CATALOG_IDX_VIEW_INDEX_HEADER_SQL = - "SELECT \": DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID,VIEW_INDEX_ID_DATA_TYPE)\" FROM %s " + "SELECT \":DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID,VIEW_INDEX_ID_DATA_TYPE)\" FROM %s " + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'"; private static RegionCoprocessorEnvironment taskRegionEnvironment; @@ -570,19 +567,6 @@ static void assertTaskColumns(Connection conn, String expectedStatus, PTable.Tas } } - private List getExplain(String query, Properties props) throws SQLException { - List explainPlan = new ArrayList<>(); - try (Connection conn = DriverManager.getConnection(getUrl(), props); - PreparedStatement statement = conn.prepareStatement("EXPLAIN " + query); - ResultSet rs = statement.executeQuery()) { - while (rs.next()) { - String plan = rs.getString(1); - explainPlan.add(plan); - } - } - return explainPlan; - } - protected PhoenixTestBuilder.SchemaBuilder createLevel2TenantViewWithGlobalLevelTTL(int globalTTL, PhoenixTestBuilder.SchemaBuilder.TenantViewOptions tenantViewOptions, PhoenixTestBuilder.SchemaBuilder.TenantViewIndexOptions tenantViewIndexOptions, @@ -823,24 +807,20 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { * Testing explain plans */ - List plans = getExplain( - "select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'v' ", - new Properties()); - assertEquals( - String.format("CLIENT PARALLEL 1-WAY FULL SCAN OVER %s", FULL_SYS_VIEW_HDR_TEST_INDEX_NAME), - plans.get(0)); - - plans = getExplain( - "select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", - new Properties()); - assertEquals(String.format("CLIENT PARALLEL 1-WAY FULL SCAN OVER %s", - FULL_SYS_VIEW_INDEX_HDR_TEST_INDEX_NAME), plans.get(0)); - - plans = getExplain( - "select ROW_KEY_MATCHER, TTL, TABLE_NAME FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL", - new Properties()); - assertEquals(String.format("CLIENT PARALLEL 1-WAY RANGE SCAN OVER %s [not null]", - FULL_SYS_ROW_KEY_MATCHER_TEST_INDEX_NAME), plans.get(0)); + try (Connection conn = DriverManager.getConnection(getUrl())) { + assertPlan(conn, + "select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'v' ") + .scanType("FULL SCAN").tableContains(FULL_SYS_VIEW_HDR_TEST_INDEX_NAME); + + assertPlan(conn, + "select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL") + .scanType("FULL SCAN").tableContains(FULL_SYS_VIEW_INDEX_HDR_TEST_INDEX_NAME); + + assertPlan(conn, + "select ROW_KEY_MATCHER, TTL, TABLE_NAME FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL") + .scanType("RANGE SCAN").tableContains(FULL_SYS_ROW_KEY_MATCHER_TEST_INDEX_NAME) + .keyRanges("[not null]"); + } /** * Testing cleanup of SYS_INDEX rows after dropping tables and views diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java index 4b8f928609f..62e0cfa5f67 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SaltedIndexIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -31,11 +32,11 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; @@ -158,15 +159,19 @@ private void testMutableTableIndexMaintanence(String dataTableName, String dataT assertEquals("y", rs.getString(2)); assertFalse(rs.next()); - String expectedPlan; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null - ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableFullName + " [~'y']\n" - + " SERVER FILTER BY FIRST KEY ONLY" - : ("CLIENT PARALLEL 4-WAY RANGE SCAN OVER " + indexTableFullName + " [X'00',~'y'] - [" - + PVarbinary.INSTANCE.toStringLiteral(new byte[] { (byte) (indexSaltBuckets - 1) }) - + ",~'y']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT"); - assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs)); + if (indexSaltBuckets == null) { + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(indexTableFullName).keyRanges("[~'y']").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); + } else { + assertPlan(conn, query).iteratorType("PARALLEL 4-WAY").scanType("RANGE SCAN") + .table(indexTableFullName) + .keyRanges("[X'00',~'y'] - [" + + PVarbinary.INSTANCE.toStringLiteral(new byte[] { (byte) (indexSaltBuckets - 1) }) + + ",~'y']") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); + } // Will use index, so rows returned in DESC order. // This is not a bug, though, because we can @@ -180,14 +185,19 @@ private void testMutableTableIndexMaintanence(String dataTableName, String dataT assertEquals("a", rs.getString(1)); assertEquals("x", rs.getString(2)); assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null - ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableFullName + " [*] - [~'x']\n" - + " SERVER FILTER BY FIRST KEY ONLY" - : ("CLIENT PARALLEL 4-WAY RANGE SCAN OVER " + indexTableFullName + " [X'00',*] - [" - + PVarbinary.INSTANCE.toStringLiteral(new byte[] { (byte) (indexSaltBuckets - 1) }) - + ",~'x']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT"); - assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs)); + if (indexSaltBuckets == null) { + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(indexTableFullName).keyRanges("[*] - [~'x']").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); + } else { + assertPlan(conn, query).iteratorType("PARALLEL 4-WAY").scanType("RANGE SCAN") + .table(indexTableFullName) + .keyRanges("[X'00',*] - [" + + PVarbinary.INSTANCE.toStringLiteral(new byte[] { (byte) (indexSaltBuckets - 1) }) + + ",~'x']") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); + } // Use data table, since point lookup trumps order by query = "SELECT k,v FROM " + dataTableFullName + " WHERE k = 'a' ORDER BY v"; @@ -196,14 +206,9 @@ private void testMutableTableIndexMaintanence(String dataTableName, String dataT assertEquals("a", rs.getString(1)); assertEquals("x", rs.getString(2)); assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = tableSaltBuckets == null - ? "CLIENT PARALLEL 1-WAY POINT LOOKUP ON 1 KEY OVER " + dataTableFullName + "\n" - + " SERVER SORTED BY [V]\n" + "CLIENT MERGE SORT" - : "CLIENT PARALLEL 1-WAY POINT LOOKUP ON 1 KEY OVER " + dataTableFullName + "\n" - + " SERVER SORTED BY [V]\n" + "CLIENT MERGE SORT"; - String explainPlan2 = QueryUtil.getExplainPlan(rs); - assertEquals(expectedPlan, explainPlan2); + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("POINT LOOKUP ON 1 KEY") + .table(dataTableFullName).serverSortedBy("[V]").clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_POINT_LOOKUP).indexRejectedNone(); // Will use data table now, since there's a LIMIT clause and // we're able to optimize out the ORDER BY, unless the data @@ -217,26 +222,29 @@ private void testMutableTableIndexMaintanence(String dataTableName, String dataT assertEquals("b", rs.getString(1)); assertEquals("y", rs.getString(2)); assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = tableSaltBuckets == null - ? "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dataTableFullName + "\n" - + " SERVER FILTER BY V >= 'x'\n" + " SERVER 2 ROW LIMIT\n" + "CLIENT 2 ROW LIMIT" - : "CLIENT PARALLEL 3-WAY FULL SCAN OVER " + dataTableFullName + "\n" - + " SERVER FILTER BY V >= 'x'\n" + " SERVER 2 ROW LIMIT\n" + "CLIENT MERGE SORT\n" - + "CLIENT 2 ROW LIMIT"; - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals(expectedPlan, explainPlan); + if (tableSaltBuckets == null) { + assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(dataTableFullName).serverWhereFilter("SERVER FILTER BY V >= 'x'").serverRowLimit(2L) + .clientRowLimit(2).indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); + } else { + assertPlan(conn, query).iteratorType("PARALLEL 3-WAY").scanType("FULL SCAN") + .table(dataTableFullName).serverWhereFilter("SERVER FILTER BY V >= 'x'").serverRowLimit(2L) + .clientSortAlgo("CLIENT MERGE SORT").clientRowLimit(2) + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); + } // PHOENIX-6604 query = "SELECT * FROM " + dataTableFullName + " ORDER BY v DESC LIMIT 1"; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - expectedPlan = indexSaltBuckets == null - ? "CLIENT SERIAL 1-WAY FULL SCAN OVER " + indexTableFullName + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER 1 ROW LIMIT\n" - + "CLIENT 1 ROW LIMIT" - : "CLIENT PARALLEL 4-WAY FULL SCAN OVER " + indexTableFullName + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER 1 ROW LIMIT\n" - + "CLIENT MERGE SORT\n" + "CLIENT 1 ROW LIMIT"; - assertEquals(expectedPlan, QueryUtil.getExplainPlan(rs)); + if (indexSaltBuckets == null) { + assertPlan(conn, query).iteratorType("SERIAL 1-WAY").scanType("FULL SCAN") + .table(indexTableFullName).serverFirstKeyOnlyProjection(true).serverRowLimit(1L) + .clientRowLimit(1).indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); + } else { + assertPlan(conn, query).iteratorType("PARALLEL 4-WAY").scanType("FULL SCAN") + .table(indexTableFullName).serverFirstKeyOnlyProjection(true).serverRowLimit(1L) + .clientSortAlgo("CLIENT MERGE SORT").clientRowLimit(1) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); + } } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SingleCellIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SingleCellIndexIT.java index 94572eabbf9..070707aac00 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SingleCellIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/SingleCellIndexIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.index; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN; import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS; import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; @@ -50,10 +51,10 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.Before; import org.junit.Test; @@ -108,11 +109,11 @@ public void testCreateOneCellTableAndSingleCellIndex() throws Exception { String selectFromData = "SELECT /*+ NO_INDEX */ PK1, INT_PK, V1, V2, V4 FROM " + tableName + " where V2 >= 3 and V4 LIKE 'V4%'"; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectFromData); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualExplainPlan.contains(tableName)); + assertPlan(conn, selectFromData).table(tableName).indexRule(OptimizerReasons.RULE_DATA_TABLE) + .indexRejectedCount(1) + .indexRejected(0, idxName, OptimizerReasons.REASON_EXCLUDED_BY_NO_INDEX_HINT); - rs = conn.createStatement().executeQuery(selectFromData); + ResultSet rs = conn.createStatement().executeQuery(selectFromData); assertTrue(rs.next()); assertEquals("PK2", rs.getString(1)); assertEquals(2, rs.getInt(2)); @@ -123,9 +124,8 @@ public void testCreateOneCellTableAndSingleCellIndex() throws Exception { String selectFromIndex = "SELECT PK1, INT_PK, V1, V2, V4 FROM " + tableName + " where V2 >= 3 and V4 LIKE 'V4%'"; - rs = conn.createStatement().executeQuery("EXPLAIN " + selectFromIndex); - actualExplainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualExplainPlan.contains(idxName)); + assertPlan(conn, selectFromIndex).table(idxName) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); rs = conn.createStatement().executeQuery(selectFromIndex); assertTrue(rs.next()); @@ -166,11 +166,10 @@ public void testAddColumns() throws Exception { String selectFromIndex = "SELECT PK1, INT_PK, V1, V2, V4, V_NEW FROM " + tableName + " where V1='V199' AND V2=100"; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectFromIndex); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualExplainPlan.contains(idxName)); + assertPlan(conn, selectFromIndex).table(idxName) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); - rs = conn.createStatement().executeQuery(selectFromIndex); + ResultSet rs = conn.createStatement().executeQuery(selectFromIndex); assertTrue(rs.next()); assertEquals("PK99", rs.getString(1)); assertEquals(99, rs.getInt(2)); @@ -207,9 +206,8 @@ public void testDropColumns() throws Exception { assertFalse(rs.next()); String selectFromIndex = "SELECT PK1, INT_PK, V1, V4 FROM " + tableName + " where V1='V11'"; - rs = conn.createStatement().executeQuery("EXPLAIN " + selectFromIndex); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualExplainPlan.contains(idxName)); + assertPlan(conn, selectFromIndex).table(idxName) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); rs = conn.createStatement().executeQuery(selectFromIndex); assertTrue(rs.next()); @@ -408,9 +406,8 @@ public void testMultipleColumnFamilies() throws Exception { String selectFromIndex = "SELECT PK1, INT_PK, A.V2, B.V3, A.V4, B.V5 FROM " + tableName + " where A.V4='V42' and B.V3 >= 3"; - rs = conn.createStatement().executeQuery("EXPLAIN " + selectFromIndex); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualExplainPlan.contains(idxName)); + assertPlan(conn, selectFromIndex).table(idxName) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); rs = conn.createStatement().executeQuery(selectFromIndex); assertTrue(rs.next()); assertEquals("PK2", rs.getString(1)); @@ -442,11 +439,11 @@ public void testPartialUpdateSingleCellTable() throws Exception { conn.commit(); String selectFromData = "SELECT /*+ NO_INDEX */ PK1, INT_PK, V1, V2, V4 FROM " + tableName + " where INT_PK = 1 and V4 LIKE 'UpdatedV4'"; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectFromData); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualExplainPlan.contains(tableName)); + assertPlan(conn, selectFromData).table(tableName).indexRule(OptimizerReasons.RULE_DATA_TABLE) + .indexRejectedCount(1) + .indexRejected(0, idxName, OptimizerReasons.REASON_EXCLUDED_BY_NO_INDEX_HINT); - rs = conn.createStatement().executeQuery(selectFromData); + ResultSet rs = conn.createStatement().executeQuery(selectFromData); assertTrue(rs.next()); assertEquals("PK1", rs.getString(1)); assertEquals(1, rs.getInt(2)); @@ -456,9 +453,8 @@ public void testPartialUpdateSingleCellTable() throws Exception { String selectFromIndex = "SELECT PK1, INT_PK, V1, V2, V4 FROM " + tableName + " where V2 >= 2 and V4 = 'UpdatedV4'"; - rs = conn.createStatement().executeQuery("EXPLAIN " + selectFromIndex); - actualExplainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualExplainPlan.contains(idxName)); + assertPlan(conn, selectFromIndex).table(idxName) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); rs = conn.createStatement().executeQuery(selectFromIndex); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java index 555f87ff0cb..d6c716e4383 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java @@ -20,6 +20,7 @@ import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.assertExplainPlan; import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.assertExplainPlanWithLimit; import static org.apache.phoenix.hbase.index.IndexRegionObserver.PHOENIX_INDEX_CDC_CONSUMER_ENABLED; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -29,6 +30,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; @@ -62,11 +64,11 @@ import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.hbase.index.IndexRegionObserver; import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.EnvironmentEdgeManager; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.TestUtil; import org.junit.After; @@ -325,7 +327,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -346,7 +348,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -363,7 +365,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -377,9 +379,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { + dataTableName + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the data table - rs = conn.createStatement().executeQuery("EXPLAIN " + noIndexQuery); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(explainPlan.contains("FULL SCAN OVER " + dataTableName)); + assertPlan(conn, noIndexQuery).scanType("FULL SCAN").table(dataTableName); rs = conn.createStatement().executeQuery(noIndexQuery); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -397,7 +397,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + " val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -426,7 +426,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -539,7 +539,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -560,7 +560,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -575,7 +575,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -587,9 +587,8 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep + dataTableName + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the data table - rs = conn.createStatement().executeQuery("EXPLAIN " + noIndexQuery); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(explainPlan.contains(salted ? "RANGE" : "FULL" + " SCAN OVER " + dataTableName)); + assertPlan(conn, noIndexQuery).scanType(salted ? "RANGE SCAN" : "FULL SCAN") + .table(dataTableName); rs = conn.createStatement().executeQuery(noIndexQuery); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -607,7 +606,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + " val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("de", rs.getString(1)); @@ -634,7 +633,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -712,6 +711,22 @@ private void assertIndexTableNotSelected(Connection conn, String dataTableName, } } + /** + * Asserts the query is served by the index, disclosing the optimizer's selection rule (and, for a + * functional index, the separate {@code matches }). + */ + private void assertIndexPlan(Connection conn, String sql, String dataTableName, + String indexTableName, String functionalExpr) throws SQLException { + String rule = sql.contains("/*+ INDEX(") + ? OptimizerReasons.RULE_HINT + : OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS; + if (functionalExpr == null) { + assertExplainPlan(conn, sql, dataTableName, indexTableName, rule); + } else { + assertExplainPlan(conn, sql, dataTableName, indexTableName, rule, functionalExpr); + } + } + @Test public void testUncoveredQuery() throws Exception { String dataTableName = generateUniqueName(); @@ -767,7 +782,7 @@ public void testUncoveredQuery() throws Exception { + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("b", rs.getString(1)); @@ -787,7 +802,7 @@ public void testUncoveredQuery() throws Exception { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0' GROUP BY val1"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -805,7 +820,7 @@ public void testUncoveredQuery() throws Exception { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0'"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(3, rs.getInt(1)); @@ -820,7 +835,7 @@ public void testUncoveredQuery() throws Exception { selectSql = "SELECT val3 from " + dataTableName + " where val1 > '0' ORDER BY val1"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -865,11 +880,11 @@ public void testPartialIndexUpdate() throws Exception { // is not included by the index table selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ val4 from " + dataTableName + " WHERE val1 = 'bc' AND val2 = 'bcdd'"; - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); } else { // Verify that an index hint is not necessary for an uncovered index selectSql = "SELECT val4 from " + dataTableName + " WHERE val1 = 'bc' AND val2 = 'bcdd'"; - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); } ResultSet rs = conn.createStatement().executeQuery(selectSql); @@ -961,7 +976,7 @@ public void testCount() throws Exception { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + "Count(v3) from " + dataTableName + " where v1 = 5"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(count, rs.getInt(1)); @@ -999,7 +1014,7 @@ public void testFailDataTableRowUpdate() throws Exception { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + "val2, val3 from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the first index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -1024,7 +1039,7 @@ public void testFailPostIndexDeleteUpdate() throws Exception { String selectSql = "SELECT id from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScannerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScannerIT.java index 641fcaa2eaf..642b25c6c22 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScannerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScannerIT.java @@ -19,6 +19,7 @@ import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.assertExplainPlan; import static org.apache.phoenix.end2end.index.GlobalIndexCheckerIT.assertExplainPlanWithLimit; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -27,6 +28,7 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; import java.util.Arrays; @@ -44,11 +46,11 @@ import org.apache.phoenix.exception.PhoenixParserException; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.hbase.index.IndexRegionObserver; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.EnvironmentEdgeManager; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -192,7 +194,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -212,7 +214,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -228,7 +230,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -241,9 +243,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { + dataTableName + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the data table - rs = conn.createStatement().executeQuery("EXPLAIN " + noIndexQuery); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(explainPlan.contains("FULL SCAN OVER " + dataTableName)); + assertPlan(conn, noIndexQuery).scanType("FULL SCAN").table(dataTableName); rs = conn.createStatement().executeQuery(noIndexQuery); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -261,7 +261,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + " val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("de", rs.getString(1)); @@ -288,7 +288,7 @@ public void testUncoveredQueryWithPhoenixRowTimestamp() throws Exception { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); @@ -375,7 +375,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -394,7 +394,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -409,7 +409,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -421,9 +421,8 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep + dataTableName + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the data table - rs = conn.createStatement().executeQuery("EXPLAIN " + noIndexQuery); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(explainPlan.contains(salted ? "RANGE" : "FULL" + " SCAN OVER " + dataTableName)); + assertPlan(conn, noIndexQuery).scanType(salted ? "RANGE SCAN" : "FULL SCAN") + .table(dataTableName); rs = conn.createStatement().executeQuery(noIndexQuery); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -441,7 +440,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + " val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("de", rs.getString(1)); @@ -468,7 +467,7 @@ public void testUncoveredQueryWithPhoenixRowTimestampAndAllPkCols() throws Excep + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); @@ -532,6 +531,22 @@ private void assertIndexTableNotSelected(Connection conn, String dataTableName, } } + /** + * Asserts the query is served by the index, disclosing the optimizer's selection rule (and, for a + * functional index, the separate {@code matches }). + */ + private void assertIndexPlan(Connection conn, String sql, String dataTableName, + String indexTableName, String functionalExpr) throws SQLException { + String rule = sql.contains("/*+ INDEX(") + ? OptimizerReasons.RULE_HINT + : OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS; + if (functionalExpr == null) { + assertExplainPlan(conn, sql, dataTableName, indexTableName, rule); + } else { + assertExplainPlan(conn, sql, dataTableName, indexTableName, rule, functionalExpr); + } + } + @Test public void testUncoveredQuery() throws Exception { String dataTableName = generateUniqueName(); @@ -585,7 +600,7 @@ public void testUncoveredQuery() throws Exception { + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("b", rs.getString(1)); @@ -605,7 +620,7 @@ public void testUncoveredQuery() throws Exception { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0' GROUP BY val1"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(2, rs.getInt(1)); @@ -619,7 +634,7 @@ public void testUncoveredQuery() throws Exception { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0'"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(3, rs.getInt(1)); @@ -634,7 +649,7 @@ public void testUncoveredQuery() throws Exception { selectSql = "SELECT val3 from " + dataTableName + " where val1 > '0' ORDER BY val1"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("abcd", rs.getString(1)); @@ -673,11 +688,11 @@ public void testPartialIndexUpdate() throws Exception { // is not included by the index table selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ val4 from " + dataTableName + " WHERE val1 = 'bc' AND val2 = 'bcdd'"; - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); } else { // Verify that an index hint is not necessary for an uncovered index selectSql = "SELECT val4 from " + dataTableName + " WHERE val1 = 'bc' AND val2 = 'bcdd'"; - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); } ResultSet rs = conn.createStatement().executeQuery(selectSql); @@ -759,7 +774,7 @@ public void testCount() throws Exception { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + "Count(v3) from " + dataTableName + " where v1 = 5"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(count, rs.getInt(1)); @@ -795,7 +810,7 @@ public void testFailDataTableRowUpdate() throws Exception { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + "val2, val3 from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the first index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("abc", rs.getString(1)); @@ -816,7 +831,7 @@ public void testFailPostIndexDeleteUpdate() throws Exception { String selectSql = "SELECT id from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); @@ -867,24 +882,18 @@ public void testPointLookup() throws Exception { // Index hint is incorrect as full index name with schema is used String sql = "SELECT /*+ INDEX(" + fullDataTableName + " " + fullIndexName + ")*/ val2, val3 from " + fullDataTableName + " WHERE id = 'a'"; - ResultSet rs = stmt.executeQuery("EXPLAIN " + sql); - String actualQueryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualQueryPlan.contains("POINT LOOKUP ON 1 KEY OVER " + fullDataTableName)); - rs = stmt.executeQuery(sql); + assertPlan(conn, sql).scanType("POINT LOOKUP ON 1 KEY").table(fullDataTableName); + ResultSet rs = stmt.executeQuery(sql); assertTrue(rs.next()); // No explicit index hint and being point lookup no index will be used sql = "SELECT val2, val3 from " + fullDataTableName + " WHERE id = 'a'"; - rs = stmt.executeQuery("EXPLAIN " + sql); - actualQueryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualQueryPlan.contains("POINT LOOKUP ON 1 KEY OVER " + fullDataTableName)); + assertPlan(conn, sql).scanType("POINT LOOKUP ON 1 KEY").table(fullDataTableName); rs = stmt.executeQuery(sql); assertTrue(rs.next()); // Index hint with point lookup over data table, still index should be used sql = "SELECT /*+ INDEX(" + fullDataTableName + " " + indexName + ")*/ val2, val3 from " + fullDataTableName + " WHERE id = 'a'"; - rs = stmt.executeQuery("EXPLAIN " + sql); - actualQueryPlan = QueryUtil.getExplainPlan(rs); - assertTrue(actualQueryPlan.contains("FULL SCAN OVER " + fullIndexName)); + assertPlan(conn, sql).scanType("FULL SCAN").table(fullIndexName); rs = stmt.executeQuery(sql); assertTrue(rs.next()); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java index b7d0e0e45da..4e3b2696bc8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java @@ -22,6 +22,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.MetaDataUtil.getViewIndexSequenceName; import static org.apache.phoenix.util.MetaDataUtil.getViewIndexSequenceSchemaName; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; @@ -63,6 +64,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PNameFactory; @@ -71,7 +73,6 @@ import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Assert; @@ -256,14 +257,13 @@ public void testMultiTenantViewLocalIndex() throws Exception { conn1.commit(); String sql = "SELECT * FROM " + fullViewName + " WHERE v2 = 100"; - ResultSet rs = conn1.prepareStatement("EXPLAIN " + sql).executeQuery(); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + fullIndexName + "(" - + SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName), isNamespaceMapped) - + ") [1,'10',100]\n" + " SERVER MERGE [0.V1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); - rs = conn1.prepareStatement(sql).executeQuery(); + assertPlan(conn1, sql).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") + .table(fullIndexName + "(" + + SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName), isNamespaceMapped) + ")") + .keyRanges("[1,'10',100]").serverMergeColumns("[0.V1]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .indexRejectedNone(); + ResultSet rs = conn1.prepareStatement(sql).executeQuery(); assertTrue(rs.next()); assertFalse(rs.next()); @@ -569,9 +569,7 @@ private void createTableForRowKeyTestsAndVerify(Connection conn, String viewPkCo String select = "SELECT " + lastViewPKCol + " FROM " + childViewName + " WHERE TEXT1='text1' LIMIT 10"; - ResultSet rs1 = conn2.createStatement().executeQuery("EXPLAIN " + select); - String actualExplainPlan = QueryUtil.getExplainPlan(rs1); - assertTrue(actualExplainPlan.contains("_IDX_" + fullTableName)); + assertPlan(conn2, select).table("_IDX_" + fullTableName); ResultSet rs = conn2.createStatement().executeQuery(select); assertTrue(rs.next()); @@ -585,11 +583,12 @@ private void assertRowCount(Connection conn, String fullTableName, String fullBa assertTrue(rs.next()); assertEquals(expectedCount, rs.getInt(1)); // Ensure that index is being used - rs = stmt.executeQuery("EXPLAIN SELECT COUNT(*) FROM " + fullTableName); if (fullBaseName != null) { // Uses index and finds correct number of rows - assertTrue(QueryUtil.getExplainPlan(rs).startsWith("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + Bytes.toString(MetaDataUtil.getViewIndexPhysicalName(Bytes.toBytes(fullBaseName))))); + assertPlan(conn, "SELECT COUNT(*) FROM " + fullTableName).iteratorType("PARALLEL 1-WAY") + .scanType("RANGE SCAN") + .table(Bytes.toString(MetaDataUtil.getViewIndexPhysicalName(Bytes.toBytes(fullBaseName)))) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone(); } // Force it not to use index and still finds correct number of rows diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java index 659f1b01ef1..95811171b8b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.index.txn; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -43,7 +44,6 @@ import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.junit.Before; @@ -159,9 +159,7 @@ private void helpTestWriteFailure(boolean indexTableWriteFailure) throws SQLExce // verify that only k3,v3 exists in the data table String dataSql = "SELECT k, v1 FROM " + dataTableFullName + " order by k"; - rs = conn.createStatement().executeQuery("EXPLAIN " + dataSql); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dataTableFullName, - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, dataSql).scanType("FULL SCAN").table(dataTableFullName); rs = conn.createStatement().executeQuery(dataSql); assertTrue(rs.next()); assertEquals("k3", rs.getString(1)); @@ -170,15 +168,10 @@ private void helpTestWriteFailure(boolean indexTableWriteFailure) throws SQLExce // verify the only k3,v3 exists in the index table String indexSql = "SELECT k, v1 FROM " + dataTableFullName + " order by v1"; - rs = conn.createStatement().executeQuery("EXPLAIN " + indexSql); if (localIndex) { - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexFullName + "(" + dataTableFullName - + ") [1]\n" + " SERVER FILTER BY EMPTY COLUMN ONLY\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, indexSql).scanType("RANGE SCAN").tableContains(indexFullName); } else { - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + indexFullName - + "\n SERVER FILTER BY EMPTY COLUMN ONLY", QueryUtil.getExplainPlan(rs)); + assertPlan(conn, indexSql).scanType("FULL SCAN").tableContains(indexFullName); } rs = conn.createStatement().executeQuery(indexSql); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/BaseJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/BaseJoinIT.java index a050c16f291..0a07e2b4300 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/BaseJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/BaseJoinIT.java @@ -18,8 +18,6 @@ package org.apache.phoenix.end2end.join; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.sql.Connection; import java.sql.Date; @@ -30,13 +28,11 @@ import java.text.SimpleDateFormat; import java.util.Map; import java.util.Properties; -import java.util.regex.Pattern; import org.apache.phoenix.cache.ServerCacheClient; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.SchemaUtil; -import org.apache.phoenix.util.StringUtil; import org.junit.Before; import org.apache.phoenix.thirdparty.com.google.common.collect.ImmutableMap; @@ -104,18 +100,15 @@ public abstract class BaseJoinIT extends ParallelStatsDisabledIT { protected String seqName; private String schemaName; protected final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - protected final String[] plans; private final String[] indexDDL; private final Map virtualNameToRealNameMap = Maps.newHashMap(); - public BaseJoinIT(String[] indexDDL, String[] plans) { + public BaseJoinIT(String[] indexDDL) { this.indexDDL = indexDDL; - this.plans = plans; } public BaseJoinIT() { this.indexDDL = new String[0]; - this.plans = new String[0]; } protected String getSchemaName() { @@ -164,33 +157,6 @@ public void createSchema() throws SQLException { } } - private String translateToVirtualPlan(String actualPlan) { - int size = getTableNameMap().size(); - String[] virtualNames = new String[size + 1]; - String[] realNames = new String[size + 1]; - int count = 0; - for (Map.Entry entry : getTableNameMap().entrySet()) { - virtualNames[count] = entry.getKey(); - realNames[count] = entry.getValue(); - count++; - } - realNames[count] = getSchemaName(); - virtualNames[count] = JOIN_SCHEMA; - String convertedPlan = StringUtil.replace(actualPlan, realNames, virtualNames); - return convertedPlan; - } - - protected void assertPlansMatch(String virtualPlanRegEx, String actualPlan) { - String convertedPlan = translateToVirtualPlan(actualPlan); - assertTrue("\"" + convertedPlan + "\" does not match \"" + virtualPlanRegEx + "\"", - Pattern.matches(virtualPlanRegEx, convertedPlan)); - } - - protected void assertPlansEqual(String virtualPlan, String actualPlan) { - String convertedPlan = translateToVirtualPlan(actualPlan); - assertEquals(virtualPlan, convertedPlan); - } - private static void initValues(Connection conn, String virtualName, String realName) throws Exception { SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinGlobalIndexIT.java index dd8f4b144c0..a27ab5d620b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinGlobalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinGlobalIndexIT.java @@ -17,10 +17,16 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; + +import java.sql.Connection; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.junit.experimental.categories.Category; import org.junit.runners.Parameterized.Parameters; @@ -46,8 +52,315 @@ protected Map getTableNameMap() { return virtualNameToRealNameMap; } - public HashJoinGlobalIndexIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public HashJoinGlobalIndexIT(String[] indexDDL) { + super(indexDDL); + } + + @Override + protected void assertLeftJoinWithAggPlan1(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.0:NAME\"]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertLeftJoinWithAggPlan2(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.:item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[SUM(O.QUANTITY) DESC]").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertLeftJoinWithAggPlan3(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]") + .clientSortedBy("[SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertRightJoinWithAggPlan1(Connection conn, String query) throws Exception { + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.0:NAME\"]") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertRightJoinWithAggPlan2(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]") + .clientSortedBy("[SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithWildcardPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(supplier).end(); + } + + @Override + protected void assertJoinPlanWithIndexPlan1(Connection conn, String query) throws Exception { + String idxItem = getSchemaName() + ".idx_item"; + String idxSupplier = getSchemaName() + ".idx_supplier"; + assertPlan(conn, query).scanType("RANGE SCAN").table(idxItem).keyRanges("['T1'] - ['T5']") + .serverFirstKeyOnlyProjection(true).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(idxSupplier).keyRanges("['S1'] - ['S5']") + .serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertJoinPlanWithIndexPlan2(Connection conn, String query) throws Exception { + String idxItem = getSchemaName() + ".idx_item"; + String idxSupplier = getSchemaName() + ".idx_supplier"; + assertPlan(conn, query).scanType("SKIP SCAN ON 2 KEYS").table(idxItem) + .keyRanges("['T1'] - ['T5']").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("SKIP SCAN ON 2 KEYS").table(idxSupplier).keyRanges("['S1'] - ['S5']") + .serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertSkipMergeOptimizationPlan(Connection conn, String query) throws Exception { + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String idxSupplier = getSchemaName() + ".idx_supplier"; + assertPlan(conn, query).scanType("FULL SCAN").table(idxItem).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .scanType("FULL SCAN").table(order).serverWhereFilter("SERVER FILTER BY QUANTITY < 5000") + .end().subPlan(1).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxSupplier).serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertSelfJoinPlan1(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + assertPlan(conn, query).scanType("FULL SCAN").table(item) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I1.item_id\" IN (\"I2.:item_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertSelfJoinPlan2(Connection conn, String query) throws Exception { + String idxItem = getSchemaName() + ".idx_item"; + assertPlan(conn, query).scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true) + .serverSortedBy("[\"I1.0:NAME\", \"I2.0:NAME\"]").clientSortAlgo("CLIENT MERGE SORT") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).end(); + } + + @Override + protected void assertStarJoinPlan(Connection conn, String query, boolean noStarJoin) + throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String idxCustomer = getSchemaName() + ".idx_customer"; + String idxItem = getSchemaName() + ".idx_item"; + if (!noStarJoin) { + assertPlan(conn, query).scanType("FULL SCAN").table(order).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxCustomer).serverFirstKeyOnlyProjection(true).end() + .subPlan(1).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true).end(); + } else { + assertPlan(conn, query).scanType("FULL SCAN").table(idxItem) + .serverFirstKeyOnlyProjection(true).serverSortedBy("[\"O.order_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxCustomer).serverFirstKeyOnlyProjection(true).end().end(); + } + } + + @Override + protected void assertSubJoinPlan(Connection conn, String query) throws Exception { + String customer = getTableName(conn, JOIN_CUSTOMER_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(customer).keyRanges("[*] - ['0000000005']") + .serverSortedBy("[\"C.customer_id\", \"I.0:NAME\"]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"C.customer_id\" IN (\"O.customer_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverWhereFilter("SERVER FILTER BY \"order_id\" != '000000000000003'").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).serverWhereFilter("SERVER FILTER BY \"NAME\" != 'T3'") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(supplier).end().end().end(); + } + + @Override + protected void assertSubqueryAggPlan1(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone().end(); + } + + @Override + protected void assertSubqueryAggPlan2(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [O.IID]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[SUM(O.QUANTITY) DESC]").subPlanCount(1) + .subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone().end(); + } + + @Override + protected void assertSubqueryAggPlan3(Connection conn, String query) throws Exception { + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true) + .serverSortedBy("[O.Q DESC NULLS LAST, I.IID]").clientSortAlgo("CLIENT MERGE SORT") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_DATA_TABLE) + .indexRejectedNone().end(); + } + + @Override + protected void assertSubqueryAggPlan4(Connection conn, String query) throws Exception { + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true) + .serverSortedBy("[O.Q DESC, I.IID]").clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_DATA_TABLE) + .indexRejectedNone().end(); + } + + @Override + protected void assertNestedSubqueriesPlan(Connection conn, String query) throws Exception { + String customer = getTableName(conn, JOIN_CUSTOMER_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(customer).keyRanges("[*] - ['0000000005']") + .serverSortedBy("[C.CID, QO.INAME]").clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverWhereFilter("SERVER FILTER BY \"order_id\" != '000000000000003'").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).serverWhereFilter("SERVER FILTER BY \"NAME\" != 'T3'") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(supplier).indexRule(OptimizerReasons.RULE_ONLY_CANDIDATE) + .indexRejectedCount(1) + .indexRejected(0, "idx_supplier", OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION).end() + .end().end(); + } + + @Override + protected void assertJoinWithLimitPlan1(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverRowLimit(4L).clientRowLimit(4).joinScannerLimit(4L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).end().subPlan(1) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithLimitPlan2(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(supplier).clientRowLimit(4) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")") + .joinScannerLimit(4L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).end().subPlan(1) + .abstractExplainPlan( + "PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertSetMaxRowsPlan(Connection conn, String query) throws Exception { + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class); + statement.setMaxRows(4); + ExplainPlanAttributes attributes = + statement.optimizeQuery().getExplainPlan().getPlanStepsAsAttributes(); + assertPlan(attributes).scanType("FULL SCAN").table(idxItem).serverFirstKeyOnlyProjection(true) + .clientRowLimit(4).joinScannerLimit(4L).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithOffsetPlan1(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverOffset(2).serverRowLimit(3L).clientRowLimit(1).joinScannerLimit(3L).subPlanCount(2) + .subPlan(0).abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).end().subPlan(1) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithOffsetPlan2(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String idxItem = getSchemaName() + ".idx_item"; + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverOffset(2).clientRowLimit(1) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")") + .joinScannerLimit(3L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(idxItem).end().subPlan(1) + .abstractExplainPlan( + "PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); } @Parameters(name = "HashJoinGlobalIndexIT_{index}") // name is used by failsafe as file name in @@ -58,278 +371,7 @@ public static synchronized Collection data() { { "CREATE INDEX \"idx_customer\" ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + " (name)", "CREATE INDEX \"idx_item\" ON " + JOIN_ITEM_TABLE_FULL_NAME + " (name) INCLUDE (price, discount1, discount2, \"supplier_id\", description)", - "CREATE INDEX \"idx_supplier\" ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + " (name)" }, - { - /* - * testLeftJoinWithAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o LEFT - * JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.0:NAME\"]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testLeftJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinOrderTable o - * LEFT JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC" - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.:item_id\"]\n" + "CLIENT MERGE SORT\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testLeftJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinItemTable i - * LEFT JOIN joinOrderTable o ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC - * NULLS LAST, iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testRightJoinWithAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o RIGHT - * JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.0:NAME\"]\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testRightJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinOrderTable - * o RIGHT JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC - * NULLS LAST, iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testJoinWithWildcard() SELECT * FROM joinItemTable LEFT JOIN joinSupplierTable supp ON - * joinItemTable.supplier_id = supp.supplier_id ORDER BY item_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME, - /* - * testJoinPlanWithIndex() SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM - * joinItemTable item LEFT JOIN joinSupplierTable supp ON substr(item.name, 2, 1) = - * substr(supp.name, 2, 1) AND (supp.name BETWEEN 'S1' AND 'S5') WHERE item.name BETWEEN - * 'T1' AND 'T5' - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_SCHEMA + ".idx_item ['T1'] - ['T5']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_SCHEMA - + ".idx_supplier ['S1'] - ['S5']\n" + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testJoinPlanWithIndex() SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM - * joinItemTable item INNER JOIN joinSupplierTable supp ON item.supplier_id = - * supp.supplier_id WHERE (item.name = 'T1' OR item.name = 'T5') AND (supp.name = 'S1' OR - * supp.name = 'S5') - */ - "CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER " + JOIN_SCHEMA - + ".idx_item ['T1'] - ['T5']\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER " + JOIN_SCHEMA - + ".idx_supplier ['S1'] - ['S5']\n" + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testJoinWithSkipMergeOptimization() SELECT s.name FROM joinItemTable i JOIN - * joinOrderTable o ON o.item_id = i.item_id AND quantity < 5000 JOIN joinSupplierTable s ON - * i.supplier_id = s.supplier_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " PARALLEL INNER-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY QUANTITY < 5000\n" + " PARALLEL INNER-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_supplier\n" - + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testSelfJoin() SELECT i2.item_id, i1.name FROM joinItemTable i1 JOIN joinItemTable i2 ON - * i1.item_id = i2.item_id ORDER BY i1.item_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " DYNAMIC SERVER FILTER BY \"I1.item_id\" IN (\"I2.:item_id\")", - /* - * testSelfJoin() SELECT i1.name, i2.name FROM joinItemTable i1 JOIN joinItemTable i2 ON - * i1.item_id = i2.supplier_id ORDER BY i1.name, i2.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I1.0:NAME\", \"I2.0:NAME\"]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_item", - /* - * testStarJoin() SELECT order_id, c.name, i.name iname, quantity, o.date FROM - * joinOrderTable o JOIN joinCustomerTable c ON o.customer_id = c.customer_id JOIN - * joinItemTable i ON o.item_id = i.item_id ORDER BY order_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_customer\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " PARALLEL INNER-JOIN TABLE 1\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testStarJoin() SELECT (*NO_STAR_JOIN*) order_id, c.name, i.name iname, quantity, o.date - * FROM joinOrderTable o JOIN joinCustomerTable c ON o.customer_id = c.customer_id JOIN - * joinItemTable i ON o.item_id = i.item_id ORDER BY order_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER SORTED BY [\"O.order_id\"]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA - + ".idx_customer\n" + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testSubJoin() SELECT * FROM joinCustomerTable c INNER JOIN (joinOrderTable o INNER JOIN - * (joinSupplierTable s RIGHT JOIN joinItemTable i ON i.supplier_id = s.supplier_id) ON - * o.item_id = i.item_id) ON c.customer_id = o.customer_id WHERE c.customer_id <= - * '0000000005' AND order_id != '000000000000003' AND i.name != 'T3' ORDER BY c.customer_id, - * i.name - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_CUSTOMER_TABLE_FULL_NAME - + " [*] - ['0000000005']\n" + " SERVER SORTED BY [\"C.customer_id\", \"I.0:NAME\"]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY \"order_id\" != '000000000000003'\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY \"NAME\" != 'T3'\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"C.customer_id\" IN (\"O.customer_id\")", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o - * LEFT JOIN (SELECT name, item_id iid FROM joinItemTable) AS i ON o.item_id = i.iid GROUP - * BY i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testJoinWithSubqueryAndAggregation() SELECT o.iid, sum(o.quantity) q FROM (SELECT item_id - * iid, quantity FROM joinOrderTable) AS o LEFT JOIN (SELECT item_id FROM joinItemTable) AS - * i ON o.iid = i.item_id GROUP BY o.iid ORDER BY q DESC - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [O.IID]\n" + "CLIENT MERGE SORT\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC]\n" - + " PARALLEL LEFT-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.iid, o.q FROM (SELECT item_id iid FROM - * joinItemTable) AS i LEFT JOIN (SELECT item_id iid, sum(quantity) q FROM joinOrderTable - * GROUP BY item_id) AS o ON o.iid = i.iid ORDER BY o.q DESC NULLS LAST, i.iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [O.Q DESC NULLS LAST, I.IID]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.iid, o.q FROM (SELECT item_id iid, - * sum(quantity) q FROM joinOrderTable GROUP BY item_id) AS o JOIN (SELECT item_id iid FROM - * joinItemTable) AS i ON o.iid = i.iid ORDER BY o.q DESC, i.iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER SORTED BY [O.Q DESC, I.IID]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - /* - * testNestedSubqueries() SELECT * FROM (SELECT customer_id cid, name, phone, address, - * loc_id, date FROM joinCustomerTable) AS c INNER JOIN (SELECT o.oid ooid, o.cid ocid, - * o.iid oiid, o.price * o.quantity, o.date odate, qi.iiid iiid, qi.iname iname, qi.iprice - * iprice, qi.idiscount1 idiscount1, qi.idiscount2 idiscount2, qi.isid isid, qi.idescription - * idescription, qi.ssid ssid, qi.sname sname, qi.sphone sphone, qi.saddress saddress, - * qi.sloc_id sloc_id FROM (SELECT item_id iid, customer_id cid, order_id oid, price, - * quantity, date FROM joinOrderTable) AS o INNER JOIN (SELECT i.iid iiid, i.name iname, - * i.price iprice, i.discount1 idiscount1, i.discount2 idiscount2, i.sid isid, i.description - * idescription, s.sid ssid, s.name sname, s.phone sphone, s.address saddress, s.loc_id - * sloc_id FROM (SELECT supplier_id sid, name, phone, address, loc_id FROM - * joinSupplierTable) AS s RIGHT JOIN (SELECT item_id iid, name, price, discount1, - * discount2, supplier_id sid, description FROM joinItemTable) AS i ON i.sid = s.sid) as qi - * ON o.iid = qi.iiid) as qo ON c.cid = qo.ocid WHERE c.cid <= '0000000005' AND qo.ooid != - * '000000000000003' AND qo.iname != 'T3' ORDER BY c.cid, qo.iname - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_CUSTOMER_TABLE_FULL_NAME - + " [*] - ['0000000005']\n" + " SERVER SORTED BY [C.CID, QO.INAME]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY \"order_id\" != '000000000000003'\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY \"NAME\" != 'T3'\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME, - /* - * testJoinWithLimit() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s LEFT JOIN joinItemTable i ON i.supplier_id = s.supplier_id LEFT JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 4 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER 4 ROW LIMIT\n" + "CLIENT 4 ROW LIMIT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " PARALLEL LEFT-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithLimit() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s JOIN joinItemTable i ON i.supplier_id = s.supplier_id JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 4 - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + "CLIENT 4 ROW LIMIT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " PARALLEL INNER-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")\n" - + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithSetMaxRows() statement.setMaxRows(4); SELECT order_id, i.name, quantity FROM - * joinItemTable i JOIN joinOrderTable o ON o.item_id = i.item_id; SELECT o.order_id, - * i.name, o.quantity FROM joinItemTable i JOIN (SELECT order_id, item_id, quantity FROM - * joinOrderTable) o ON o.item_id = i.item_id; - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT 4 ROW LIMIT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithLimit() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s LEFT JOIN joinItemTable i ON i.supplier_id = s.supplier_id LEFT JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 1 OFFSET 2 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER OFFSET 2\n" + " SERVER 3 ROW LIMIT\n" + "CLIENT 1 ROW LIMIT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_item\n" + " PARALLEL LEFT-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " JOIN-SCANNER 3 ROW LIMIT", - /* - * testJoinWithLimit() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s JOIN joinItemTable i ON i.supplier_id = s.supplier_id JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 1 OFFSET 2 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER OFFSET 2\n" + "CLIENT 1 ROW LIMIT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " PARALLEL INNER-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")\n" - + " JOIN-SCANNER 3 ROW LIMIT", } }); + "CREATE INDEX \"idx_supplier\" ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + " (name)" } }); return testCases; } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinIT.java index cfbc0c94504..7105a29dd55 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinIT.java @@ -40,7 +40,6 @@ import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -48,10 +47,327 @@ @RunWith(Parameterized.class) public abstract class HashJoinIT extends BaseJoinIT { - public HashJoinIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public HashJoinIT(String[] indexDDL) { + super(indexDDL); } + /* + * The expected EXPLAIN plan for each of the queries below differs per index configuration, so + * each concrete subclass supplies the attribute-based assertions via these hooks. + */ + + /** + * {@link #testLeftJoinWithAggregation()}: + * + * 
+   * SELECT i.name, sum(quantity) FROM joinOrderTable o
+   *   LEFT JOIN joinItemTable i ON o.item_id = i.item_id
+   *   GROUP BY i.name ORDER BY i.name
+   *
+ */ + protected abstract void assertLeftJoinWithAggPlan1(Connection conn, String query) + throws Exception; + + /** + * {@link #testLeftJoinWithAggregation()}: + * + * 
+   * SELECT i.item_id iid, sum(quantity) q FROM joinOrderTable o
+   *   LEFT JOIN joinItemTable i ON o.item_id = i.item_id
+   *   GROUP BY i.item_id ORDER BY q DESC
+   *
+ */ + protected abstract void assertLeftJoinWithAggPlan2(Connection conn, String query) + throws Exception; + + /** + * {@link #testLeftJoinWithAggregation()}: + * + * 
+   * SELECT i.item_id iid, sum(quantity) q FROM joinItemTable i
+   *   LEFT JOIN joinOrderTable o ON o.item_id = i.item_id
+   *   GROUP BY i.item_id ORDER BY q DESC NULLS LAST, iid
+   *
+ */ + protected abstract void assertLeftJoinWithAggPlan3(Connection conn, String query) + throws Exception; + + /** + * {@link #testRightJoinWithAggregation()}: + * + * 
+   * SELECT i.name, sum(quantity) FROM joinOrderTable o
+   *   RIGHT JOIN joinItemTable i ON o.item_id = i.item_id
+   *   GROUP BY i.name ORDER BY i.name
+   *
+ */ + protected abstract void assertRightJoinWithAggPlan1(Connection conn, String query) + throws Exception; + + /** + * {@link #testRightJoinWithAggregation()}: + * + * 
+   * SELECT i.item_id iid, sum(quantity) q FROM joinOrderTable o
+   *   RIGHT JOIN joinItemTable i ON o.item_id = i.item_id
+   *   GROUP BY i.item_id ORDER BY q DESC NULLS LAST, iid
+   *
+ */ + protected abstract void assertRightJoinWithAggPlan2(Connection conn, String query) + throws Exception; + + /** + * {@link #testJoinWithWildcard()}: + * + * 
+   * SELECT * FROM joinItemTable
+   *   LEFT JOIN joinSupplierTable supp ON joinItemTable.supplier_id = supp.supplier_id
+   *   ORDER BY item_id
+   *
+ */ + protected abstract void assertJoinWithWildcardPlan(Connection conn, String query) + throws Exception; + + /** + * {@link #testJoinPlanWithIndex()}: + * + * 
+   * SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM joinItemTable item
+   *   LEFT JOIN joinSupplierTable supp
+   *     ON substr(item.name, 2, 1) = substr(supp.name, 2, 1)
+   *     AND (supp.name BETWEEN 'S1' AND 'S5')
+   *   WHERE item.name BETWEEN 'T1' AND 'T5'
+   *
+ */ + protected abstract void assertJoinPlanWithIndexPlan1(Connection conn, String query) + throws Exception; + + /** + * {@link #testJoinPlanWithIndex()}: + * + * 
+   * SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM joinItemTable item
+   *   INNER JOIN joinSupplierTable supp ON item.supplier_id = supp.supplier_id
+   *   WHERE (item.name = 'T1' OR item.name = 'T5')
+   *     AND (supp.name = 'S1' OR supp.name = 'S5')
+   *
+ */ + protected abstract void assertJoinPlanWithIndexPlan2(Connection conn, String query) + throws Exception; + + /** + * {@link #testJoinWithSkipMergeOptimization()}: + * + * 
+   * SELECT s.name FROM joinItemTable i
+   *   JOIN joinOrderTable o ON o.item_id = i.item_id AND quantity < 5000
+   *   JOIN joinSupplierTable s ON i.supplier_id = s.supplier_id
+   *
+ */ + protected abstract void assertSkipMergeOptimizationPlan(Connection conn, String query) + throws Exception; + + /** + * {@link #testSelfJoin()}: + * + * 
+   * SELECT i2.item_id, i1.name FROM joinItemTable i1
+   *   JOIN joinItemTable i2 ON i1.item_id = i2.item_id
+   *   ORDER BY i1.item_id
+   *
+ */ + protected abstract void assertSelfJoinPlan1(Connection conn, String query) throws Exception; + + /** + * {@link #testSelfJoin()}: + * + * 
+   * SELECT i1.name, i2.name FROM joinItemTable i1
+   *   JOIN joinItemTable i2 ON i1.item_id = i2.supplier_id
+   *   ORDER BY i1.name, i2.name
+   *
+ */ + protected abstract void assertSelfJoinPlan2(Connection conn, String query) throws Exception; + + /** + * {@link #testStarJoin()}. {@code noStarJoin} selects the {@code NO_STAR_JOIN} variant. + * + * 
+   * SELECT order_id, c.name, i.name iname, quantity, o.date FROM joinOrderTable o
+   *   JOIN joinCustomerTable c ON o.customer_id = c.customer_id
+   *   JOIN joinItemTable i ON o.item_id = i.item_id
+   *   ORDER BY order_id
+   *
+   * -- noStarJoin variant adds the NO_STAR_JOIN hint:
+   * SELECT /*+ NO_STAR_JOIN*/ order_id, c.name, i.name iname, quantity, o.date
+   *   FROM joinOrderTable o
+   *   JOIN joinCustomerTable c ON o.customer_id = c.customer_id
+   *   JOIN joinItemTable i ON o.item_id = i.item_id
+   *   ORDER BY order_id
+   *
+ */ + protected abstract void assertStarJoinPlan(Connection conn, String query, boolean noStarJoin) + throws Exception; + + /** + * {@link #testSubJoin()}: + * + * 
+   * SELECT * FROM joinCustomerTable c
+   *   INNER JOIN (joinOrderTable o
+   *     INNER JOIN (joinSupplierTable s
+   *       RIGHT JOIN joinItemTable i ON i.supplier_id = s.supplier_id)
+   *     ON o.item_id = i.item_id)
+   *   ON c.customer_id = o.customer_id
+   *   WHERE c.customer_id <= '0000000005'
+   *     AND order_id != '000000000000003' AND i.name != 'T3'
+   *   ORDER BY c.customer_id, i.name
+   *
+ */ + protected abstract void assertSubJoinPlan(Connection conn, String query) throws Exception; + + /** + * {@link #testJoinWithSubqueryAndAggregation()}: + * + * 
+   * SELECT i.name, sum(quantity) FROM joinOrderTable o
+   *   LEFT JOIN (SELECT name, item_id iid FROM joinItemTable) AS i ON o.item_id = i.iid
+   *   GROUP BY i.name ORDER BY i.name
+   *
+ */ + protected abstract void assertSubqueryAggPlan1(Connection conn, String query) throws Exception; + + /** + * {@link #testJoinWithSubqueryAndAggregation()}: + * + * 
+   * SELECT o.iid, sum(o.quantity) q
+   *   FROM (SELECT item_id iid, quantity FROM joinOrderTable) AS o
+   *   LEFT JOIN (SELECT item_id FROM joinItemTable) AS i ON o.iid = i.item_id
+   *   GROUP BY o.iid ORDER BY q DESC
+   *
+ */ + protected abstract void assertSubqueryAggPlan2(Connection conn, String query) throws Exception; + + /** + * {@link #testJoinWithSubqueryAndAggregation()}: + * + * 
+   * SELECT i.iid, o.q
+   *   FROM (SELECT item_id iid FROM joinItemTable) AS i
+   *   LEFT JOIN (SELECT item_id iid, sum(quantity) q FROM joinOrderTable GROUP BY item_id) AS o
+   *     ON o.iid = i.iid
+   *   ORDER BY o.q DESC NULLS LAST, i.iid
+   *
+ */ + protected abstract void assertSubqueryAggPlan3(Connection conn, String query) throws Exception; + + /** + * {@link #testJoinWithSubqueryAndAggregation()}: + * + * 
+   * SELECT i.iid, o.q
+   *   FROM (SELECT item_id iid, sum(quantity) q FROM joinOrderTable GROUP BY item_id) AS o
+   *   JOIN (SELECT item_id iid FROM joinItemTable) AS i ON o.iid = i.iid
+   *   ORDER BY o.q DESC, i.iid
+   *
+ */ + protected abstract void assertSubqueryAggPlan4(Connection conn, String query) throws Exception; + + /** + * {@link #testNestedSubqueries()}: + * + * 
+   * SELECT * FROM
+   *   (SELECT customer_id cid, name, phone, address, loc_id, date FROM joinCustomerTable) AS c
+   *   INNER JOIN
+   *   (SELECT o.oid ooid, o.cid ocid, o.iid oiid, o.price * o.quantity, o.date odate,
+   *           qi.iiid iiid, qi.iname iname, qi.iprice iprice, qi.idiscount1 idiscount1,
+   *           qi.idiscount2 idiscount2, qi.isid isid, qi.idescription idescription,
+   *           qi.ssid ssid, qi.sname sname, qi.sphone sphone, qi.saddress saddress,
+   *           qi.sloc_id sloc_id
+   *      FROM (SELECT item_id iid, customer_id cid, order_id oid, price, quantity, date
+   *              FROM joinOrderTable) AS o
+   *      INNER JOIN
+   *      (SELECT i.iid iiid, i.name iname, i.price iprice, i.discount1 idiscount1,
+   *              i.discount2 idiscount2, i.sid isid, i.description idescription,
+   *              s.sid ssid, s.name sname, s.phone sphone, s.address saddress, s.loc_id sloc_id
+   *         FROM (SELECT supplier_id sid, name, phone, address, loc_id FROM joinSupplierTable) AS s
+   *         RIGHT JOIN (SELECT item_id iid, name, price, discount1, discount2, supplier_id sid,
+   *                            description FROM joinItemTable) AS i ON i.sid = s.sid) as qi
+   *      ON o.iid = qi.iiid) as qo
+   *   ON c.cid = qo.ocid
+   *   WHERE c.cid <= '0000000005' AND qo.ooid != '000000000000003' AND qo.iname != 'T3'
+   *   ORDER BY c.cid, qo.iname
+   *
+ */ + protected abstract void assertNestedSubqueriesPlan(Connection conn, String query) + throws Exception; + + /** + * {@link #testJoinWithLimit()}: + * + * 
+   * SELECT order_id, i.name, s.name, s.address, quantity FROM joinSupplierTable s
+   *   LEFT JOIN joinItemTable i ON i.supplier_id = s.supplier_id
+   *   LEFT JOIN joinOrderTable o ON o.item_id = i.item_id
+   *   LIMIT 4
+   *
+ */ + protected abstract void assertJoinWithLimitPlan1(Connection conn, String query) throws Exception; + + /** + * {@link #testJoinWithLimit()}: + * + * 
+   * SELECT order_id, i.name, s.name, s.address, quantity FROM joinSupplierTable s
+   *   JOIN joinItemTable i ON i.supplier_id = s.supplier_id
+   *   JOIN joinOrderTable o ON o.item_id = i.item_id
+   *   LIMIT 4
+   *
+ */ + protected abstract void assertJoinWithLimitPlan2(Connection conn, String query) throws Exception; + + /** + * Assert the EXPLAIN plan for {@link #testJoinWithSetMaxRows()} (with a max-rows limit of 4). The + * {@code CLIENT 4 ROW LIMIT} comes from {@link java.sql.Statement#setMaxRows(int)} rather than + * the SQL, so subclasses must compile via a {@code PhoenixPreparedStatement}. + * + * 
+   * statement.setMaxRows(4);
+   * SELECT order_id, i.name, quantity FROM joinItemTable i
+   *   JOIN joinOrderTable o ON o.item_id = i.item_id
+   *
+   * SELECT o.order_id, i.name, o.quantity FROM joinItemTable i
+   *   JOIN (SELECT order_id, item_id, quantity FROM joinOrderTable) o ON o.item_id = i.item_id
+   *
+ */ + protected abstract void assertSetMaxRowsPlan(Connection conn, String query) throws Exception; + + /** + * {@link #testJoinWithOffset()}: + * + * 
+   * SELECT order_id, i.name, s.name, s.address, quantity FROM joinSupplierTable s
+   *   LEFT JOIN joinItemTable i ON i.supplier_id = s.supplier_id
+   *   LEFT JOIN joinOrderTable o ON o.item_id = i.item_id
+   *   LIMIT 1 OFFSET 2
+   *
+ */ + protected abstract void assertJoinWithOffsetPlan1(Connection conn, String query) throws Exception; + + /** + * {@link #testJoinWithOffset()}: + * + * 
+   * SELECT order_id, i.name, s.name, s.address, quantity FROM joinSupplierTable s
+   *   JOIN joinItemTable i ON i.supplier_id = s.supplier_id
+   *   JOIN joinOrderTable o ON o.item_id = i.item_id
+   *   LIMIT 1 OFFSET 2
+   *
+ */ + protected abstract void assertJoinWithOffsetPlan2(Connection conn, String query) throws Exception; + public void testInnerJoin(boolean renamePhysicalTable) throws Exception { Connection conn = getConnection(); String tableName1 = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); @@ -536,8 +852,7 @@ public void testStarJoin() throws Exception { assertFalse(rs.next()); if (i < 4) { - rs = conn.createStatement().executeQuery("EXPLAIN " + query[i]); - assertPlansEqual(plans[11 + (i / 2)], QueryUtil.getExplainPlan(rs)); + assertStarJoinPlan(conn, query[i], i >= 2); } } } finally { @@ -575,8 +890,7 @@ public void testLeftJoinWithAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query1); - assertPlansEqual(plans[0], QueryUtil.getExplainPlan(rs)); + assertLeftJoinWithAggPlan1(conn, query1); statement = conn.prepareStatement(query2); rs = statement.executeQuery(); @@ -595,8 +909,7 @@ public void testLeftJoinWithAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[1], QueryUtil.getExplainPlan(rs)); + assertLeftJoinWithAggPlan2(conn, query2); statement = conn.prepareStatement(query3); rs = statement.executeQuery(); @@ -624,8 +937,7 @@ public void testLeftJoinWithAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query3); - assertPlansEqual(plans[2], QueryUtil.getExplainPlan(rs)); + assertLeftJoinWithAggPlan3(conn, query3); } finally { conn.close(); } @@ -668,8 +980,7 @@ public void testRightJoinWithAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query1); - assertPlansEqual(plans[3], QueryUtil.getExplainPlan(rs)); + assertRightJoinWithAggPlan1(conn, query1); statement = conn.prepareStatement(query2); rs = statement.executeQuery(); @@ -697,8 +1008,7 @@ public void testRightJoinWithAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[4], QueryUtil.getExplainPlan(rs)); + assertRightJoinWithAggPlan2(conn, query2); } finally { conn.close(); } @@ -1181,8 +1491,7 @@ public void testJoinWithWildcard() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[5], QueryUtil.getExplainPlan(rs)); + assertJoinWithWildcardPlan(conn, query); } finally { conn.close(); } @@ -1490,8 +1799,7 @@ public void testJoinPlanWithIndex() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query1); - assertPlansEqual(plans[6], QueryUtil.getExplainPlan(rs)); + assertJoinPlanWithIndexPlan1(conn, query1); statement = conn.prepareStatement(query2); rs = statement.executeQuery(); @@ -1508,8 +1816,7 @@ public void testJoinPlanWithIndex() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[7], QueryUtil.getExplainPlan(rs)); + assertJoinPlanWithIndexPlan2(conn, query2); } finally { conn.close(); } @@ -1537,8 +1844,7 @@ public void testJoinWithSkipMergeOptimization() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[8], QueryUtil.getExplainPlan(rs)); + assertSkipMergeOptimizationPlan(conn, query); } finally { conn.close(); } @@ -1581,8 +1887,7 @@ public void testSelfJoin() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query1); - assertPlansEqual(plans[9], QueryUtil.getExplainPlan(rs)); + assertSelfJoinPlan1(conn, query1); statement = conn.prepareStatement(query2); rs = statement.executeQuery(); @@ -1607,8 +1912,7 @@ public void testSelfJoin() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[10], QueryUtil.getExplainPlan(rs)); + assertSelfJoinPlan2(conn, query2); } finally { conn.close(); } @@ -1889,8 +2193,7 @@ public void testSubJoin() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[13], QueryUtil.getExplainPlan(rs)); + assertSubJoinPlan(conn, query2); } finally { conn.close(); } @@ -2057,8 +2360,7 @@ public void testJoinWithSubqueryAndAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query1); - assertPlansEqual(plans[14], QueryUtil.getExplainPlan(rs)); + assertSubqueryAggPlan1(conn, query1); statement = conn.prepareStatement(query2); rs = statement.executeQuery(); @@ -2077,8 +2379,7 @@ public void testJoinWithSubqueryAndAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[15], QueryUtil.getExplainPlan(rs)); + assertSubqueryAggPlan2(conn, query2); statement = conn.prepareStatement(query3); rs = statement.executeQuery(); @@ -2106,8 +2407,7 @@ public void testJoinWithSubqueryAndAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query3); - assertPlansEqual(plans[16], QueryUtil.getExplainPlan(rs)); + assertSubqueryAggPlan3(conn, query3); statement = conn.prepareStatement(query4); rs = statement.executeQuery(); @@ -2126,8 +2426,7 @@ public void testJoinWithSubqueryAndAggregation() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query4); - assertPlansEqual(plans[17], QueryUtil.getExplainPlan(rs)); + assertSubqueryAggPlan4(conn, query4); } finally { conn.close(); } @@ -2263,8 +2562,7 @@ public void testNestedSubqueries() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[18], QueryUtil.getExplainPlan(rs)); + assertNestedSubqueriesPlan(conn, query2); } finally { conn.close(); } @@ -2315,8 +2613,7 @@ public void testJoinWithLimit() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query1); - assertPlansEqual(plans[19], QueryUtil.getExplainPlan(rs)); + assertJoinWithLimitPlan1(conn, query1); statement = conn.prepareStatement(query2); rs = statement.executeQuery(); @@ -2347,8 +2644,7 @@ public void testJoinWithLimit() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[20], QueryUtil.getExplainPlan(rs)); + assertJoinWithLimitPlan2(conn, query2); } finally { conn.close(); } @@ -2381,8 +2677,7 @@ public void testJoinWithOffset() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query1); - assertPlansEqual(plans[22], QueryUtil.getExplainPlan(rs)); + assertJoinWithOffsetPlan1(conn, query1); statement = conn.prepareStatement(query2); rs = statement.executeQuery(); @@ -2395,8 +2690,7 @@ public void testJoinWithOffset() throws Exception { assertEquals(rs.getInt(5), 5000); assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query2); - assertPlansEqual(plans[23], QueryUtil.getExplainPlan(rs)); + assertJoinWithOffsetPlan2(conn, query2); } finally { conn.close(); } @@ -2499,8 +2793,7 @@ public void testJoinWithSetMaxRows() throws Exception { assertFalse(rs.next()); - rs = statement.executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[21], QueryUtil.getExplainPlan(rs)); + assertSetMaxRowsPlan(conn, query); } } finally { conn.close(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinLocalIndexIT.java index 0dbe28fe898..b16cb480536 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinLocalIndexIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -30,9 +31,12 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; +import org.apache.phoenix.util.SchemaUtil; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -62,8 +66,370 @@ protected Map getTableNameMap() { return virtualNameToRealNameMap; } - public HashJoinLocalIndexIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public HashJoinLocalIndexIT(String[] indexDDL) { + super(indexDDL); + } + + @Override + protected void assertLeftJoinWithAggPlan1(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.0:NAME\"]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertLeftJoinWithAggPlan2(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.:item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[SUM(O.QUANTITY) DESC]").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertLeftJoinWithAggPlan3(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]") + .clientSortedBy("[SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertRightJoinWithAggPlan1(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.0:NAME\"]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertRightJoinWithAggPlan2(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]") + .clientSortedBy("[SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithWildcardPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(supplier).end(); + } + + @Override + protected void assertJoinPlanWithIndexPlan1(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String supplierIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_SUPPLIER_INDEX); + assertPlan(conn, query).scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1,'T1'] - [1,'T5']").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(supplierIndex + "(" + supplier + ")") + .keyRanges("[1,'S1'] - [1,'S5']").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertJoinPlanWithIndexPlan2(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String supplierIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_SUPPLIER_INDEX); + assertPlan(conn, query).scanType("SKIP SCAN ON 2 KEYS").table(itemIndex + "(" + item + ")") + .keyRanges("[1,'T1'] - [1,'T5']").clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("SKIP SCAN ON 2 KEYS").table(supplierIndex + "(" + supplier + ")") + .keyRanges("[1,'S1'] - [1,'S5']").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertSkipMergeOptimizationPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String supplierIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_SUPPLIER_INDEX); + assertPlan(conn, query).scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I.:item_id\" IN (\"O.item_id\")") + .subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .scanType("FULL SCAN").table(order).serverWhereFilter("SERVER FILTER BY QUANTITY < 5000") + .end().subPlan(1).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(supplierIndex + "(" + supplier + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertSelfJoinPlan1(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + assertPlan(conn, query).scanType("FULL SCAN").table(item) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I1.item_id\" IN (\"I2.:item_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertSelfJoinPlan2(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + assertPlan(conn, query).scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true) + .serverSortedBy("[\"I1.0:NAME\", \"I2.0:NAME\"]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I1.:item_id\" IN (\"I2.0:supplier_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertStarJoinPlan(Connection conn, String query, boolean noStarJoin) + throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String customer = getTableName(conn, JOIN_CUSTOMER_TABLE_FULL_NAME); + String customerIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_CUSTOMER_INDEX); + if (!noStarJoin) { + assertPlan(conn, query).scanType("FULL SCAN").table(order).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(customerIndex + "(" + customer + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT").end().subPlan(1) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT").end(); + } else { + assertPlan(conn, query).scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true).serverSortedBy("[\"O.order_id\"]") + .clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I.:item_id\" IN (\"O.item_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(customerIndex + "(" + customer + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT").end().end(); + } + } + + @Override + protected void assertSubJoinPlan(Connection conn, String query) throws Exception { + String customer = getTableName(conn, JOIN_CUSTOMER_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(customer).keyRanges("[*] - ['0000000005']") + .serverSortedBy("[\"C.customer_id\", \"I.0:NAME\"]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"C.customer_id\" IN (\"O.customer_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverWhereFilter("SERVER FILTER BY \"order_id\" != '000000000000003'").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverWhereFilter("SERVER FILTER BY \"NAME\" != 'T3'").clientSortAlgo("CLIENT MERGE SORT") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(supplier).end().end().end(); + } + + @Override + protected void assertSubqueryAggPlan1(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone().end(); + } + + @Override + protected void assertSubqueryAggPlan2(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [O.IID]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[SUM(O.QUANTITY) DESC]").subPlanCount(1) + .subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED).indexRejectedNone().end(); + } + + @Override + protected void assertSubqueryAggPlan3(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true) + .serverSortedBy("[O.Q DESC NULLS LAST, I.IID]").clientSortAlgo("CLIENT MERGE SORT") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_DATA_TABLE) + .indexRejectedNone().end(); + } + + @Override + protected void assertSubqueryAggPlan4(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true).serverSortedBy("[O.Q DESC, I.IID]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").indexRule(OptimizerReasons.RULE_DATA_TABLE) + .indexRejectedNone().end(); + } + + @Override + protected void assertNestedSubqueriesPlan(Connection conn, String query) throws Exception { + String customer = getTableName(conn, JOIN_CUSTOMER_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(customer).keyRanges("[*] - ['0000000005']") + .serverSortedBy("[C.CID, QO.INAME]").clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverWhereFilter("SERVER FILTER BY \"order_id\" != '000000000000003'").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverWhereFilter("SERVER FILTER BY \"NAME\" != 'T3'").clientSortAlgo("CLIENT MERGE SORT") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(supplier).indexRule(OptimizerReasons.RULE_NON_LOCAL_PREFERRED) + .indexRejectedCount(1).indexRejected(0, JOIN_SUPPLIER_INDEX, + OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE) + .end().end().end(); + } + + @Override + protected void assertJoinWithLimitPlan1(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverRowLimit(4L).clientRowLimit(4).joinScannerLimit(4L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").end().subPlan(1) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithLimitPlan2(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(supplier).clientRowLimit(4) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")") + .joinScannerLimit(4L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").end().subPlan(1) + .abstractExplainPlan( + "PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertSetMaxRowsPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class); + statement.setMaxRows(4); + ExplainPlanAttributes attributes = + statement.optimizeQuery().getExplainPlan().getPlanStepsAsAttributes(); + assertPlan(attributes).scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true).clientSortAlgo("CLIENT MERGE SORT") + .clientRowLimit(4) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I.:item_id\" IN (\"O.item_id\")") + .joinScannerLimit(4L).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithOffsetPlan1(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverOffset(2).serverRowLimit(3L).clientRowLimit(1).joinScannerLimit(3L).subPlanCount(2) + .subPlan(0).abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").end().subPlan(1) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithOffsetPlan2(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverOffset(2).clientRowLimit(1) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")") + .joinScannerLimit(3L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .clientSortAlgo("CLIENT MERGE SORT").end().subPlan(1) + .abstractExplainPlan( + "PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); } @Parameters(name = "HashJoinLocalIndexIT_{index}") // name is used by failsafe as file name in @@ -76,347 +442,7 @@ public static synchronized Collection data() { "CREATE LOCAL INDEX " + JOIN_ITEM_INDEX + " ON " + JOIN_ITEM_TABLE_FULL_NAME + " (name) " + "INCLUDE (price, discount1, discount2, \"supplier_id\", description)", "CREATE LOCAL INDEX " + JOIN_SUPPLIER_INDEX + " ON " + JOIN_SUPPLIER_TABLE_FULL_NAME - + " (name)" }, - { - /* - * testLeftJoinWithAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o LEFT - * JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.0:NAME\"]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " CLIENT MERGE SORT", - /* - * testLeftJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinOrderTable o - * LEFT JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC" - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.:item_id\"]\n" + "CLIENT MERGE SORT\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " CLIENT MERGE SORT", - /* - * testLeftJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinItemTable i - * LEFT JOIN joinOrderTable o ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC - * NULLS LAST, iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testRightJoinWithAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o RIGHT - * JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.0:NAME\"]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testRightJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinOrderTable - * o RIGHT JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC - * NULLS LAST, iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testJoinWithWildcard() SELECT * FROM joinItemTable LEFT JOIN joinSupplierTable supp ON - * joinItemTable.supplier_id = supp.supplier_id ORDER BY item_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME, - /* - * testJoinPlanWithIndex() SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM - * joinItemTable item LEFT JOIN joinSupplierTable supp ON substr(item.name, 2, 1) = - * substr(supp.name, 2, 1) AND (supp.name BETWEEN 'S1' AND 'S5') WHERE item.name BETWEEN - * 'T1' AND 'T5' - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1,'T1'] - [1,'T5']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" + JOIN_SUPPLIER_TABLE_FULL_NAME - + ") [1,'S1'] - [1,'S5']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " CLIENT MERGE SORT", - /* - * testJoinPlanWithIndex() SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM - * joinItemTable item INNER JOIN joinSupplierTable supp ON item.supplier_id = - * supp.supplier_id WHERE (item.name = 'T1' OR item.name = 'T5') AND (supp.name = 'S1' OR - * supp.name = 'S5') - */ - "CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1,'T1'] - [1,'T5']\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER " - + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" + JOIN_SUPPLIER_TABLE_FULL_NAME - + ") [1,'S1'] - [1,'S5']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " CLIENT MERGE SORT", - /* - * testJoinWithSkipMergeOptimization() SELECT s.name FROM joinItemTable i JOIN - * joinOrderTable o ON o.item_id = i.item_id AND quantity < 5000 JOIN joinSupplierTable s ON - * i.supplier_id = s.supplier_id - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY QUANTITY < 5000\n" + " PARALLEL INNER-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" - + JOIN_SUPPLIER_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I.:item_id\" IN (\"O.item_id\")", - /* - * testSelfJoin() SELECT i2.item_id, i1.name FROM joinItemTable i1 JOIN joinItemTable i2 ON - * i1.item_id = i2.item_id ORDER BY i1.item_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I1.item_id\" IN (\"I2.:item_id\")", - /* - * testSelfJoin() SELECT i1.name, i2.name FROM joinItemTable i1 JOIN joinItemTable i2 ON - * i1.item_id = i2.supplier_id ORDER BY i1.name, i2.name - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I1.0:NAME\", \"I2.0:NAME\"]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I1.:item_id\" IN (\"I2.0:supplier_id\")", - /* - * testStarJoin() SELECT order_id, c.name, i.name iname, quantity, o.date FROM - * joinOrderTable o JOIN joinCustomerTable c ON o.customer_id = c.customer_id JOIN - * joinItemTable i ON o.item_id = i.item_id ORDER BY order_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_CUSTOMER_INDEX_FULL_NAME + "(" + JOIN_CUSTOMER_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 1\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " CLIENT MERGE SORT", - /* - * testStarJoin() SELECT (*NO_STAR_JOIN*) order_id, c.name, i.name iname, quantity, o.date - * FROM joinOrderTable o JOIN joinCustomerTable c ON o.customer_id = c.customer_id JOIN - * joinItemTable i ON o.item_id = i.item_id ORDER BY order_id - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"O.order_id\"]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_CUSTOMER_INDEX_FULL_NAME - + "(" + JOIN_CUSTOMER_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I.:item_id\" IN (\"O.item_id\")", - /* - * testSubJoin() SELECT * FROM joinCustomerTable c INNER JOIN (joinOrderTable o INNER JOIN - * (joinSupplierTable s RIGHT JOIN joinItemTable i ON i.supplier_id = s.supplier_id) ON - * o.item_id = i.item_id) ON c.customer_id = o.customer_id WHERE c.customer_id <= - * '0000000005' AND order_id != '000000000000003' AND i.name != 'T3' ORDER BY c.customer_id, - * i.name - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_CUSTOMER_TABLE_FULL_NAME - + " [*] - ['0000000005']\n" + " SERVER SORTED BY [\"C.customer_id\", \"I.0:NAME\"]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY \"order_id\" != '000000000000003'\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME - + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY \"NAME\" != 'T3'\n" - + " CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"C.customer_id\" IN (\"O.customer_id\")", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o - * LEFT JOIN (SELECT name, item_id iid FROM joinItemTable) AS i ON o.item_id = i.iid GROUP - * BY i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " CLIENT MERGE SORT", - /* - * testJoinWithSubqueryAndAggregation() SELECT o.iid, sum(o.quantity) q FROM (SELECT item_id - * iid, quantity FROM joinOrderTable) AS o LEFT JOIN (SELECT item_id FROM joinItemTable) AS - * i ON o.iid = i.item_id GROUP BY o.iid ORDER BY q DESC - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [O.IID]\n" + "CLIENT MERGE SORT\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC]\n" - + " PARALLEL LEFT-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " CLIENT MERGE SORT", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.iid, o.q FROM (SELECT item_id iid FROM - * joinItemTable) AS i LEFT JOIN (SELECT item_id iid, sum(quantity) q FROM joinOrderTable - * GROUP BY item_id) AS o ON o.iid = i.iid ORDER BY o.q DESC NULLS LAST, i.iid - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [O.Q DESC NULLS LAST, I.IID]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.iid, o.q FROM (SELECT item_id iid, - * sum(quantity) q FROM joinOrderTable GROUP BY item_id) AS o JOIN (SELECT item_id iid FROM - * joinItemTable) AS i ON o.iid = i.iid ORDER BY o.q DESC, i.iid - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [O.Q DESC, I.IID]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - /* - * testNestedSubqueries() SELECT * FROM (SELECT customer_id cid, name, phone, address, - * loc_id, date FROM joinCustomerTable) AS c INNER JOIN (SELECT o.oid ooid, o.cid ocid, - * o.iid oiid, o.price * o.quantity, o.date odate, qi.iiid iiid, qi.iname iname, qi.iprice - * iprice, qi.idiscount1 idiscount1, qi.idiscount2 idiscount2, qi.isid isid, qi.idescription - * idescription, qi.ssid ssid, qi.sname sname, qi.sphone sphone, qi.saddress saddress, - * qi.sloc_id sloc_id FROM (SELECT item_id iid, customer_id cid, order_id oid, price, - * quantity, date FROM joinOrderTable) AS o INNER JOIN (SELECT i.iid iiid, i.name iname, - * i.price iprice, i.discount1 idiscount1, i.discount2 idiscount2, i.sid isid, i.description - * idescription, s.sid ssid, s.name sname, s.phone sphone, s.address saddress, s.loc_id - * sloc_id FROM (SELECT supplier_id sid, name, phone, address, loc_id FROM - * joinSupplierTable) AS s RIGHT JOIN (SELECT item_id iid, name, price, discount1, - * discount2, supplier_id sid, description FROM joinItemTable) AS i ON i.sid = s.sid) as qi - * ON o.iid = qi.iiid) as qo ON c.cid = qo.ocid WHERE c.cid <= '0000000005' AND qo.ooid != - * '000000000000003' AND qo.iname != 'T3' ORDER BY c.cid, qo.iname - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_CUSTOMER_TABLE_FULL_NAME - + " [*] - ['0000000005']\n" + " SERVER SORTED BY [C.CID, QO.INAME]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY \"order_id\" != '000000000000003'\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME - + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY \"NAME\" != 'T3'\n" - + " CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME, - /* - * testJoinWithLimit() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s LEFT JOIN joinItemTable i ON i.supplier_id = s.supplier_id LEFT JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 4 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER 4 ROW LIMIT\n" + "CLIENT 4 ROW LIMIT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithLimit() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s JOIN joinItemTable i ON i.supplier_id = s.supplier_id JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 4 - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + "CLIENT 4 ROW LIMIT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")\n" - + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithSetMaxRows() statement.setMaxRows(4); SELECT order_id, i.name, quantity FROM - * joinItemTable i JOIN joinOrderTable o ON o.item_id = i.item_id; SELECT o.order_id, - * i.name, o.quantity FROM joinItemTable i JOIN (SELECT order_id, item_id, quantity FROM - * joinOrderTable) o ON o.item_id = i.item_id; - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + "CLIENT MERGE SORT\n" + "CLIENT 4 ROW LIMIT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"I.:item_id\" IN (\"O.item_id\")\n" - + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithOffset() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s LEFT JOIN joinItemTable i ON i.supplier_id = s.supplier_id LEFT JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 1 OFFSET 2 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER OFFSET 2\n" + " SERVER 3 ROW LIMIT\n" + "CLIENT 1 ROW LIMIT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " JOIN-SCANNER 3 ROW LIMIT", - /* - * testJoinWithOffset() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s JOIN joinItemTable i ON i.supplier_id = s.supplier_id JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 1 OFFSET 2 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER OFFSET 2\n" + "CLIENT 1 ROW LIMIT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.0:supplier_id\")\n" - + " JOIN-SCANNER 3 ROW LIMIT", - /* - * testJoinWithLocalIndex() SELECT phone, i.name FROM joinSupplierTable s JOIN joinItemTable - * i ON s.supplier_id = i.supplier_id WHERE s.name = 'S1' AND i.name < 'T6' - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" - + JOIN_SUPPLIER_TABLE_FULL_NAME + ") [1,'S1']\n" + " SERVER MERGE [0.PHONE]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1,*] - [1,'T6']\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"S.:supplier_id\" IN (\"I.0:supplier_id\")", - - /* - * testJoinWithLocalIndex() SELECT phone, max(i.name) FROM joinSupplierTable s JOIN - * joinItemTable i ON s."supplier_id" = i."supplier_id" WHERE s.name = 'S1' AND i.name < - * 'T6' GROUP BY phone - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" - + JOIN_SUPPLIER_TABLE_FULL_NAME + ") [1,'S1']\n" + " SERVER MERGE [0.PHONE]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"S.PHONE\"]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1,*] - [1,'T6']\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"S.:supplier_id\" IN (\"I.0:supplier_id\")", - - /* - * testJoinWithLocalIndex() SELECT max(phone), max(i.name) FROM joinSupplierTable s LEFT - * JOIN joinItemTable i ON s."supplier_id" = i."supplier_id" AND i.name < 'T6' WHERE s.name - * <= 'S3' - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" - + JOIN_SUPPLIER_TABLE_FULL_NAME + ") [1,*] - [1,'S3']\n" + " SERVER MERGE [0.PHONE]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO SINGLE ROW\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1,*] - [1,'T6']\n" - + " CLIENT MERGE SORT", } }); + + " (name)" } }); return testCases; } @@ -425,10 +451,13 @@ public void testJoinWithLocalIndex() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); try { - String query = - "select phone, i.name from " + getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME) - + " s join " + getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME) - + " i on s.\"supplier_id\" = i.\"supplier_id\" where s.name = 'S1' and i.name < 'T6'"; + String supplierTable = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String itemTable = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplierIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_SUPPLIER_INDEX); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + + String query = "select phone, i.name from " + supplierTable + " s join " + itemTable + + " i on s.\"supplier_id\" = i.\"supplier_id\" where s.name = 'S1' and i.name < 'T6'"; System.out.println("1)\n" + query); PreparedStatement statement = conn.prepareStatement(query); ResultSet rs = statement.executeQuery(); @@ -437,11 +466,17 @@ public void testJoinWithLocalIndex() throws Exception { assertTrue(rs.next()); assertEquals(rs.getString(1), "888-888-1111"); assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[24], QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).scanType("RANGE SCAN") + .table(supplierIndex + "(" + supplierTable + ")").keyRanges("[1,'S1']") + .serverMergeColumns("[0.PHONE]").serverFirstKeyOnlyProjection(true) + .clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"S.:supplier_id\" IN (\"I.0:supplier_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + itemTable + ")") + .keyRanges("[1,*] - [1,'T6']").clientSortAlgo("CLIENT MERGE SORT").end(); - query = "select phone, max(i.name) from " + getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME) - + " s join " + getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME) + query = "select phone, max(i.name) from " + supplierTable + " s join " + itemTable + " i on s.\"supplier_id\" = i.\"supplier_id\" where s.name = 'S1' and i.name < 'T6' group by phone"; statement = conn.prepareStatement(query); rs = statement.executeQuery(); @@ -449,21 +484,32 @@ public void testJoinWithLocalIndex() throws Exception { assertEquals(rs.getString(1), "888-888-1111"); assertEquals(rs.getString(2), "T2"); assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[25], QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).scanType("RANGE SCAN") + .table(supplierIndex + "(" + supplierTable + ")").keyRanges("[1,'S1']") + .serverMergeColumns("[0.PHONE]").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"S.PHONE\"]") + .clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"S.:supplier_id\" IN (\"I.0:supplier_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + itemTable + ")") + .keyRanges("[1,*] - [1,'T6']").clientSortAlgo("CLIENT MERGE SORT").end(); - query = - "select max(phone), max(i.name) from " + getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME) - + " s left join " + getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME) - + " i on s.\"supplier_id\" = i.\"supplier_id\" and i.name < 'T6' where s.name <= 'S3'"; + query = "select max(phone), max(i.name) from " + supplierTable + " s left join " + itemTable + + " i on s.\"supplier_id\" = i.\"supplier_id\" and i.name < 'T6' where s.name <= 'S3'"; statement = conn.prepareStatement(query); rs = statement.executeQuery(); assertTrue(rs.next()); assertEquals(rs.getString(1), "888-888-3333"); assertEquals(rs.getString(2), "T4"); assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[26], QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).scanType("RANGE SCAN") + .table(supplierIndex + "(" + supplierTable + ")").keyRanges("[1,*] - [1,'S3']") + .serverMergeColumns("[0.PHONE]").serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO SINGLE ROW").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table(itemIndex + "(" + itemTable + ")") + .keyRanges("[1,*] - [1,'T6']").clientSortAlgo("CLIENT MERGE SORT").end(); } finally { conn.close(); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinMoreIT.java index 85d04a10efe..73c50b6cf78 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinMoreIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinMoreIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -33,48 +34,12 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @Category(ParallelStatsDisabledTest.class) public class HashJoinMoreIT extends ParallelStatsDisabledIT { - private final String[] plans = new String[] { - /* - * testJoinWithKeyRangeOptimization() SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, - * rhs.col2 FROM TEMP_TABLE_COMPOSITE_PK lhs JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col1 = - * rhs.col2 - */ - "CLIENT PARALLEL 4-WAY FULL SCAN OVER %s\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 4-WAY FULL SCAN OVER %s\n" - + " CLIENT MERGE SORT", - /* - * testJoinWithKeyRangeOptimization() SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, - * rhs.col2 FROM TEMP_TABLE_COMPOSITE_PK lhs JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = - * rhs.col2 - */ - "CLIENT PARALLEL 4-WAY FULL SCAN OVER %s\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 4-WAY FULL SCAN OVER %s\n" - + " CLIENT MERGE SORT\n" + " DYNAMIC SERVER FILTER BY LHS.COL0 IN (RHS.COL2)", - /* - * testJoinWithKeyRangeOptimization() SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, - * rhs.col2 FROM TEMP_TABLE_COMPOSITE_PK lhs JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = - * rhs.col1 AND lhs.col1 = rhs.col2 - */ - "CLIENT PARALLEL 4-WAY FULL SCAN OVER %s\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 4-WAY FULL SCAN OVER %s\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1) IN ((RHS.COL1, RHS.COL2))", - /* - * testJoinWithKeyRangeOptimization() SELECT lhs.col0, lhs.col1, lhs.col2, rhs.col0, rhs.col1, - * rhs.col2 FROM TEMP_TABLE_COMPOSITE_PK lhs JOIN TEMP_TABLE_COMPOSITE_PK rhs ON lhs.col0 = - * rhs.col1 AND lhs.col2 = rhs.col3 - 1 AND lhs.col1 = rhs.col2 - */ - "CLIENT PARALLEL 4-WAY FULL SCAN OVER %s\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 4-WAY FULL SCAN OVER %s\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1, LHS.COL2) IN ((RHS.COL1, RHS.COL2, TO_INTEGER((RHS.COL3 - 1))))", }; @Test public void testJoinOverSaltedTables() throws Exception { @@ -327,9 +292,11 @@ public void testJoinWithKeyRangeOptimization() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(String.format(plans[0], tempTableWithCompositePK, tempTableWithCompositePK), - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).scanType("FULL SCAN").table(tempTableWithCompositePK) + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(tempTableWithCompositePK).clientSortAlgo("CLIENT MERGE SORT") + .end(); // Two parts of PK but only one leading part query = @@ -350,9 +317,12 @@ public void testJoinWithKeyRangeOptimization() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(String.format(plans[1], tempTableWithCompositePK, tempTableWithCompositePK), - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).scanType("FULL SCAN").table(tempTableWithCompositePK) + .clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY LHS.COL0 IN (RHS.COL2)").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(tempTableWithCompositePK).clientSortAlgo("CLIENT MERGE SORT") + .end(); // Two leading parts of PK query = @@ -382,9 +352,14 @@ public void testJoinWithKeyRangeOptimization() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(String.format(plans[2], tempTableWithCompositePK, tempTableWithCompositePK), - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).scanType("FULL SCAN").table(tempTableWithCompositePK) + .clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter( + "DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1) IN ((RHS.COL1, RHS.COL2))") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(tempTableWithCompositePK).clientSortAlgo("CLIENT MERGE SORT") + .end(); // All parts of PK query = @@ -414,9 +389,14 @@ public void testJoinWithKeyRangeOptimization() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertEquals(String.format(plans[3], tempTableWithCompositePK, tempTableWithCompositePK), - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).scanType("FULL SCAN").table(tempTableWithCompositePK) + .clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter( + "DYNAMIC SERVER FILTER BY (LHS.COL0, LHS.COL1, LHS.COL2) IN ((RHS.COL1, RHS.COL2, TO_INTEGER((RHS.COL3 - 1))))") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(tempTableWithCompositePK).clientSortAlgo("CLIENT MERGE SORT") + .end(); } finally { conn.close(); } @@ -800,28 +780,22 @@ public void testBug2894() throws Exception { + " ON L.BUCKET = E.BUCKET AND L.\"TIMESTAMP\" = E.\"TIMESTAMP\"\n" + " ) C\n" + " GROUP BY C.BUCKET, C.\"TIMESTAMP\""; - String p = i == 0 - ? "CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"E.TIMESTAMP\", E.BUCKET]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] - + " [X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']\n" - + " SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" - + " CLIENT MERGE SORT" - : "CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER EVENT_COUNT [X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"E.TIMESTAMP\", E.BUCKET]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 2-WAY SKIP SCAN ON 2 RANGES OVER " + t[i] - + " [X'00','5SEC',1462993420000000001,'Tr/Bal'] - [X'01','5SEC',1462993520000000000,'Tr/Bal']\n" - + " SERVER FILTER BY FIRST KEY ONLY AND SRC_LOCATION = DST_LOCATION\n" - + " CLIENT MERGE SORT"; - - ResultSet rs = conn.createStatement().executeQuery("explain " + q); - assertEquals(p, QueryUtil.getExplainPlan(rs)); - - rs = conn.createStatement().executeQuery(q); + String innerKeyRanges = i == 0 + ? "[X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']" + : "[X'00','5SEC',1462993420000000001,'Tr/Bal'] - [X'01','5SEC',1462993520000000000,'Tr/Bal']"; + + assertPlan(conn, q).scanType("SKIP SCAN ON 2 RANGES").table("EVENT_COUNT").keyRanges( + "[X'00','5SEC',~1462993520000000000,'Tr/Bal'] - [X'01','5SEC',~1462993420000000000,'Tr/Bal']") + .serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"E.TIMESTAMP\", E.BUCKET]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .scanType("SKIP SCAN ON 2 RANGES").table(t[i]).keyRanges(innerKeyRanges) + .serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY SRC_LOCATION = DST_LOCATION") + .clientSortAlgo("CLIENT MERGE SORT").end(); + + ResultSet rs = conn.createStatement().executeQuery(q); assertTrue(rs.next()); assertEquals("5SEC", rs.getString(1)); assertEquals(1462993520000000000L, rs.getLong(2)); @@ -920,12 +894,8 @@ public void testBug2961() throws Exception { + "FROM ( SELECT ACCOUNT_ID, BUCKET_ID, OBJECT_ID, MAX(OBJECT_VERSION) AS MAXVER " + " FROM test2961 GROUP BY ACCOUNT_ID, BUCKET_ID, OBJECT_ID) AS X " + " INNER JOIN test2961 AS OBJ ON X.ACCOUNT_ID = OBJ.ACCOUNT_ID AND X.BUCKET_ID = OBJ.BUCKET_ID AND X.OBJECT_ID = OBJ.OBJECT_ID AND X.MAXVER = OBJ.OBJECT_VERSION"; - rs = conn.createStatement().executeQuery("explain " + q); - String plan = QueryUtil.getExplainPlan(rs); - String dynamicFilter = - "DYNAMIC SERVER FILTER BY (OBJ.ACCOUNT_ID, OBJ.BUCKET_ID, OBJ.OBJECT_ID, OBJ.OBJECT_VERSION) IN ((X.ACCOUNT_ID, X.BUCKET_ID, X.OBJECT_ID, X.MAXVER))"; - assertTrue("Expected '" + dynamicFilter + "' to be used for the query, but got:\n" + plan, - plan.contains(dynamicFilter)); + assertPlan(conn, q).dynamicServerFilter( + "DYNAMIC SERVER FILTER BY (OBJ.ACCOUNT_ID, OBJ.BUCKET_ID, OBJ.OBJECT_ID, OBJ.OBJECT_VERSION) IN ((X.ACCOUNT_ID, X.BUCKET_ID, X.OBJECT_ID, X.MAXVER))"); rs = conn.createStatement().executeQuery(q); assertTrue(rs.next()); assertEquals("2222", rs.getString(4)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinNoIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinNoIndexIT.java index da637f0c1f0..78aee0e2bb2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinNoIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/HashJoinNoIndexIT.java @@ -17,10 +17,15 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; + +import java.sql.Connection; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.junit.experimental.categories.Category; import org.junit.runners.Parameterized.Parameters; @@ -46,283 +51,319 @@ protected Map getTableNameMap() { return virtualNameToRealNameMap; } - public HashJoinNoIndexIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public HashJoinNoIndexIT(String[] indexDDL) { + super(indexDDL); + } + + @Override + protected void assertLeftJoinWithAggPlan1(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).end(); + } + + @Override + protected void assertLeftJoinWithAggPlan2(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[SUM(O.QUANTITY) DESC]").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertLeftJoinWithAggPlan3(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]") + .clientSortedBy("[SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertRightJoinWithAggPlan1(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertRightJoinWithAggPlan2(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true) + .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]") + .clientSortedBy("[SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithWildcardPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(supplier).end(); + } + + @Override + protected void assertJoinPlanWithIndexPlan1(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item) + .serverWhereFilter("SERVER FILTER BY (NAME >= 'T1' AND NAME <= 'T5')").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(supplier) + .serverWhereFilter("SERVER FILTER BY (NAME >= 'S1' AND NAME <= 'S5')").end(); + } + + @Override + protected void assertJoinPlanWithIndexPlan2(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item) + .serverWhereFilter("SERVER FILTER BY (NAME = 'T1' OR NAME = 'T5')").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(supplier) + .serverWhereFilter("SERVER FILTER BY (NAME = 'S1' OR NAME = 'S5')").end(); + } + + @Override + protected void assertSkipMergeOptimizationPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I.item_id\" IN (\"O.item_id\")") + .subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .scanType("FULL SCAN").table(order).serverWhereFilter("SERVER FILTER BY QUANTITY < 5000") + .end().subPlan(1).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(supplier).end(); + } + + @Override + protected void assertSelfJoinPlan1(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I1.item_id\" IN (\"I2.item_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertSelfJoinPlan2(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverSortedBy("[I1.NAME, I2.NAME]") + .clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I1.item_id\" IN (\"I2.supplier_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).end(); + } + + @Override + protected void assertStarJoinPlan(Connection conn, String query, boolean noStarJoin) + throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String customer = getTableName(conn, JOIN_CUSTOMER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + if (!noStarJoin) { + assertPlan(conn, query).scanType("FULL SCAN").table(order).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(customer).end().subPlan(1) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).end(); + } else { + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverSortedBy("[\"O.order_id\"]") + .clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I.item_id\" IN (\"O.item_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(customer).end().end(); + } + } + + @Override + protected void assertSubJoinPlan(Connection conn, String query) throws Exception { + String customer = getTableName(conn, JOIN_CUSTOMER_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(customer).keyRanges("[*] - ['0000000005']") + .serverSortedBy("[\"C.customer_id\", I.NAME]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"C.customer_id\" IN (\"O.customer_id\")") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverWhereFilter("SERVER FILTER BY \"order_id\" != '000000000000003'").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).serverWhereFilter("SERVER FILTER BY NAME != 'T3'") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(supplier).end().end().end(); + } + + @Override + protected void assertSubqueryAggPlan1(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).end(); + } + + @Override + protected void assertSubqueryAggPlan2(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [O.IID]") + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[SUM(O.QUANTITY) DESC]").subPlanCount(1) + .subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT, SKIP MERGE */") + .scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true).end(); + } + + @Override + protected void assertSubqueryAggPlan3(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true) + .serverSortedBy("[O.Q DESC NULLS LAST, I.IID]").clientSortAlgo("CLIENT MERGE SORT") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertSubqueryAggPlan4(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(item).serverFirstKeyOnlyProjection(true) + .serverSortedBy("[O.Q DESC, I.IID]").clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(order) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertNestedSubqueriesPlan(Connection conn, String query) throws Exception { + String customer = getTableName(conn, JOIN_CUSTOMER_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("RANGE SCAN").table(customer).keyRanges("[*] - ['0000000005']") + .serverSortedBy("[C.CID, QO.INAME]").clientSortAlgo("CLIENT MERGE SORT").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order) + .serverWhereFilter("SERVER FILTER BY \"order_id\" != '000000000000003'").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).serverWhereFilter("SERVER FILTER BY NAME != 'T3'") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(supplier).end().end().end(); + } + + @Override + protected void assertJoinWithLimitPlan1(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverRowLimit(4L).clientRowLimit(4).joinScannerLimit(4L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).end().subPlan(1) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithLimitPlan2(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).scanType("FULL SCAN").table(supplier).clientRowLimit(4) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.supplier_id\")") + .joinScannerLimit(4L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).end().subPlan(1) + .abstractExplainPlan( + "PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertSetMaxRowsPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class); + statement.setMaxRows(4); + ExplainPlanAttributes attributes = + statement.optimizeQuery().getExplainPlan().getPlanStepsAsAttributes(); + assertPlan(attributes).scanType("FULL SCAN").table(item).clientRowLimit(4) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"I.item_id\" IN (\"O.item_id\")") + .joinScannerLimit(4L).subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithOffsetPlan1(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverOffset(2).serverRowLimit(3L).clientRowLimit(1).joinScannerLimit(3L).subPlanCount(2) + .subPlan(0).abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).end().subPlan(1) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); + } + + @Override + protected void assertJoinWithOffsetPlan2(Connection conn, String query) throws Exception { + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).iteratorType("SERIAL").scanType("FULL SCAN").table(supplier) + .serverOffset(2).clientRowLimit(1) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.supplier_id\")") + .joinScannerLimit(3L).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(item).end().subPlan(1) + .abstractExplainPlan( + "PARALLEL INNER-JOIN TABLE 1 /* HASH BUILD RIGHT, DELAYED EVALUATION */") + .scanType("FULL SCAN").table(order).end(); } @Parameters(name = "HashJoinNoIndexIT_{index}") // name is used by failsafe as file name in // reports public static synchronized Collection data() { List testCases = Lists.newArrayList(); - testCases.add(new String[][] { {}, { - /* - * testLeftJoinWithAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o LEFT JOIN - * joinItemTable i ON o.item_id = i.item_id GROUP BY i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME, - /* - * testLeftJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinOrderTable o - * LEFT JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC" - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"I.item_id\"]\n" + "CLIENT MERGE SORT\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC]\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testLeftJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinItemTable i - * LEFT JOIN joinOrderTable o ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC - * NULLS LAST, iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testRightJoinWithAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o RIGHT - * JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testRightJoinWithAggregation() SELECT i.item_id iid, sum(quantity) q FROM joinOrderTable o - * RIGHT JOIN joinItemTable i ON o.item_id = i.item_id GROUP BY i.item_id ORDER BY q DESC - * NULLS LAST, iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"I.item_id\"]\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC NULLS LAST, \"I.item_id\"]\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME, - /* - * testJoinWithWildcard() SELECT * FROM joinItemTable LEFT JOIN joinSupplierTable supp ON - * joinItemTable.supplier_id = supp.supplier_id ORDER BY item_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME, - /* - * testJoinPlanWithIndex() SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM - * joinItemTable item LEFT JOIN joinSupplierTable supp ON substr(item.name, 2, 1) = - * substr(supp.name, 2, 1) AND (supp.name BETWEEN 'S1' AND 'S5') WHERE item.name BETWEEN 'T1' - * AND 'T5' - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY (NAME >= 'T1' AND NAME <= 'T5')\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY (NAME >= 'S1' AND NAME <= 'S5')", - /* - * testJoinPlanWithIndex() SELECT item.item_id, item.name, supp.supplier_id, supp.name FROM - * joinItemTable item INNER JOIN joinSupplierTable supp ON item.supplier_id = supp.supplier_id - * WHERE (item.name = 'T1' OR item.name = 'T5') AND (supp.name = 'S1' OR supp.name = 'S5') - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY (NAME = 'T1' OR NAME = 'T5')\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY (NAME = 'S1' OR NAME = 'S5')", - /* - * testJoinWithSkipMergeOptimization() SELECT s.name FROM joinItemTable i JOIN joinOrderTable - * o ON o.item_id = i.item_id AND quantity < 5000 JOIN joinSupplierTable s ON i.supplier_id = - * s.supplier_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY QUANTITY < 5000\n" + " PARALLEL INNER-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"I.item_id\" IN (\"O.item_id\")", - /* - * testSelfJoin() SELECT i2.item_id, i1.name FROM joinItemTable i1 JOIN joinItemTable i2 ON - * i1.item_id = i2.item_id ORDER BY i1.item_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " DYNAMIC SERVER FILTER BY \"I1.item_id\" IN (\"I2.item_id\")", - /* - * testSelfJoin() SELECT i1.name, i2.name FROM joinItemTable i1 JOIN joinItemTable i2 ON - * i1.item_id = i2.supplier_id ORDER BY i1.name, i2.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER SORTED BY [I1.NAME, I2.NAME]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"I1.item_id\" IN (\"I2.supplier_id\")", - /* - * testStarJoin() SELECT order_id, c.name, i.name iname, quantity, o.date FROM joinOrderTable - * o JOIN joinCustomerTable c ON o.customer_id = c.customer_id JOIN joinItemTable i ON - * o.item_id = i.item_id ORDER BY order_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_CUSTOMER_TABLE_FULL_NAME + "\n" + " PARALLEL INNER-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME, - /* - * testStarJoin() SELECT (*NO_STAR_JOIN*) order_id, c.name, i.name iname, quantity, o.date - * FROM joinOrderTable o JOIN joinCustomerTable c ON o.customer_id = c.customer_id JOIN - * joinItemTable i ON o.item_id = i.item_id ORDER BY order_id - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER SORTED BY [\"O.order_id\"]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_CUSTOMER_TABLE_FULL_NAME - + "\n" + " DYNAMIC SERVER FILTER BY \"I.item_id\" IN (\"O.item_id\")", - /* - * testSubJoin() SELECT * FROM joinCustomerTable c INNER JOIN (joinOrderTable o INNER JOIN - * (joinSupplierTable s RIGHT JOIN joinItemTable i ON i.supplier_id = s.supplier_id) ON - * o.item_id = i.item_id) ON c.customer_id = o.customer_id WHERE c.customer_id <= '0000000005' - * AND order_id != '000000000000003' AND i.name != 'T3' ORDER BY c.customer_id, i.name - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_CUSTOMER_TABLE_FULL_NAME - + " [*] - ['0000000005']\n" + " SERVER SORTED BY [\"C.customer_id\", I.NAME]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY \"order_id\" != '000000000000003'\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY NAME != 'T3'\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"C.customer_id\" IN (\"O.customer_id\")", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.name, sum(quantity) FROM joinOrderTable o - * LEFT JOIN (SELECT name, item_id iid FROM joinItemTable) AS i ON o.item_id = i.iid GROUP BY - * i.name ORDER BY i.name - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [I.NAME]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME, - /* - * testJoinWithSubqueryAndAggregation() SELECT o.iid, sum(o.quantity) q FROM (SELECT item_id - * iid, quantity FROM joinOrderTable) AS o LEFT JOIN (SELECT item_id FROM joinItemTable) AS i - * ON o.iid = i.item_id GROUP BY o.iid ORDER BY q DESC - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [O.IID]\n" + "CLIENT MERGE SORT\n" - + "CLIENT SORTED BY [SUM(O.QUANTITY) DESC]\n" - + " PARALLEL LEFT-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.iid, o.q FROM (SELECT item_id iid FROM - * joinItemTable) AS i LEFT JOIN (SELECT item_id iid, sum(quantity) q FROM joinOrderTable - * GROUP BY item_id) AS o ON o.iid = i.iid ORDER BY o.q DESC NULLS LAST, i.iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [O.Q DESC NULLS LAST, I.IID]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - /* - * testJoinWithSubqueryAndAggregation() SELECT i.iid, o.q FROM (SELECT item_id iid, - * sum(quantity) q FROM joinOrderTable GROUP BY item_id) AS o JOIN (SELECT item_id iid FROM - * joinItemTable) AS i ON o.iid = i.iid ORDER BY o.q DESC, i.iid - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER SORTED BY [O.Q DESC, I.IID]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - /* - * testNestedSubqueries() SELECT * FROM (SELECT customer_id cid, name, phone, address, loc_id, - * date FROM joinCustomerTable) AS c INNER JOIN (SELECT o.oid ooid, o.cid ocid, o.iid oiid, - * o.price * o.quantity, o.date odate, qi.iiid iiid, qi.iname iname, qi.iprice iprice, - * qi.idiscount1 idiscount1, qi.idiscount2 idiscount2, qi.isid isid, qi.idescription - * idescription, qi.ssid ssid, qi.sname sname, qi.sphone sphone, qi.saddress saddress, - * qi.sloc_id sloc_id FROM (SELECT item_id iid, customer_id cid, order_id oid, price, - * quantity, date FROM joinOrderTable) AS o INNER JOIN (SELECT i.iid iiid, i.name iname, - * i.price iprice, i.discount1 idiscount1, i.discount2 idiscount2, i.sid isid, i.description - * idescription, s.sid ssid, s.name sname, s.phone sphone, s.address saddress, s.loc_id - * sloc_id FROM (SELECT supplier_id sid, name, phone, address, loc_id FROM joinSupplierTable) - * AS s RIGHT JOIN (SELECT item_id iid, name, price, discount1, discount2, supplier_id sid, - * description FROM joinItemTable) AS i ON i.sid = s.sid) as qi ON o.iid = qi.iiid) as qo ON - * c.cid = qo.ocid WHERE c.cid <= '0000000005' AND qo.ooid != '000000000000003' AND qo.iname - * != 'T3' ORDER BY c.cid, qo.iname - */ - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_CUSTOMER_TABLE_FULL_NAME - + " [*] - ['0000000005']\n" + " SERVER SORTED BY [C.CID, QO.INAME]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY \"order_id\" != '000000000000003'\n" - + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY NAME != 'T3'\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME, - /* - * testJoinWithLimit() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s LEFT JOIN joinItemTable i ON i.supplier_id = s.supplier_id LEFT JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 4 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER 4 ROW LIMIT\n" + "CLIENT 4 ROW LIMIT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL LEFT-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithLimit() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s JOIN joinItemTable i ON i.supplier_id = s.supplier_id JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 4 - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + "CLIENT 4 ROW LIMIT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.supplier_id\")\n" - + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithSetMaxRows() statement.setMaxRows(4); SELECT order_id, i.name, quantity FROM - * joinItemTable i JOIN joinOrderTable o ON o.item_id = i.item_id; SELECT o.order_id, i.name, - * o.quantity FROM joinItemTable i JOIN (SELECT order_id, item_id, quantity FROM - * joinOrderTable) o ON o.item_id = i.item_id; - */ - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + "CLIENT 4 ROW LIMIT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"I.item_id\" IN (\"O.item_id\")\n" - + " JOIN-SCANNER 4 ROW LIMIT", - /* - * testJoinWithOffset() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s LEFT JOIN joinItemTable i ON i.supplier_id = s.supplier_id LEFT JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 1 OFFSET 2 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER OFFSET 2\n" + " SERVER 3 ROW LIMIT\n" + "CLIENT 1 ROW LIMIT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" + " PARALLEL LEFT-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " JOIN-SCANNER 3 ROW LIMIT", - /* - * testJoinWithOffset() SELECT order_id, i.name, s.name, s.address, quantity FROM - * joinSupplierTable s JOIN joinItemTable i ON i.supplier_id = s.supplier_id JOIN - * joinOrderTable o ON o.item_id = i.item_id LIMIT 1 OFFSET 2 - */ - "CLIENT SERIAL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER OFFSET 2\n" + "CLIENT 1 ROW LIMIT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL INNER-JOIN TABLE 1(DELAYED EVALUATION)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " DYNAMIC SERVER FILTER BY \"S.supplier_id\" IN (\"I.supplier_id\")\n" - + " JOIN-SCANNER 3 ROW LIMIT", } }); + testCases.add(new String[][] { {} }); return testCases; } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinGlobalIndexIT.java index 5d3a88cafe2..a0ee9bc4905 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinGlobalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinGlobalIndexIT.java @@ -17,10 +17,15 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; + +import java.sql.Connection; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.junit.experimental.categories.Category; import org.junit.runners.Parameterized.Parameters; @@ -46,44 +51,62 @@ protected Map getTableNameMap() { return virtualNameToRealNameMap; } - public SortMergeJoinGlobalIndexIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public SortMergeJoinGlobalIndexIT(String[] indexDDL) { + super(indexDDL); + } + + @Override + protected void assertSkipMergeOptimizationPlan(Connection conn, String query) throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String itemIndex = getSchemaName() + ".idx_item"; + String supplierIndex = getSchemaName() + ".idx_supplier"; + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (LEFT)").sortMergeSkipMerge(false) + .lhs().scanType("FULL SCAN").table(supplierIndex).serverFirstKeyOnlyProjection(true) + .serverSortedBy("[\"S.:supplier_id\"]").clientSortAlgo("CLIENT MERGE SORT").end().rhs() + .abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(true) + .clientSortedBy("[\"I.0:supplier_id\"]").lhs().scanType("FULL SCAN").table(itemIndex) + .serverSortedBy("[\"I.:item_id\"]").clientSortAlgo("CLIENT MERGE SORT").end().rhs() + .scanType("FULL SCAN").table(order).serverWhereFilter("SERVER FILTER BY QUANTITY < 5000") + .serverSortedBy("[\"O.item_id\"]").clientSortAlgo("CLIENT MERGE SORT").end().end(); + } + + @Override + protected void assertSelfJoinPlan(Connection conn, String query) throws Exception { + String itemIndex = getSchemaName() + ".idx_item"; + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(false) + .lhs().scanType("FULL SCAN").table(itemIndex).serverFirstKeyOnlyProjection(true) + .serverSortedBy("[\"I1.:item_id\"]").clientSortAlgo("CLIENT MERGE SORT").end().rhs() + .scanType("FULL SCAN").table(itemIndex).serverFirstKeyOnlyProjection(true) + .serverSortedBy("[\"I2.:item_id\"]").clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertSetMaxRowsPlan(Connection conn, String query, int queryIndex) + throws Exception { + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String itemIndex = getSchemaName() + ".idx_item"; + PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class); + statement.setMaxRows(4); + ExplainPlanAttributes attributes = + statement.optimizeQuery().getExplainPlan().getPlanStepsAsAttributes(); + assertPlan(attributes).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(false) + .clientRowLimit(4).lhs().scanType("FULL SCAN").table(itemIndex) + .serverFirstKeyOnlyProjection(true).serverSortedBy("[\"I.:item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end().rhs().scanType("FULL SCAN").table(order) + .serverSortedBy(queryIndex == 0 ? "[\"O.item_id\"]" : "[\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end(); } - @Parameters(name = "SortMergeJoinGlobalIndexIT_{index}") // name is used by failsafe as file name - // in reports + // name is used by failsafe as file name in reports + @Parameters(name = "SortMergeJoinGlobalIndexIT_{index}") public static synchronized Collection data() { List testCases = Lists.newArrayList(); testCases.add(new String[][] { { "CREATE INDEX \"idx_customer\" ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + " (name)", "CREATE INDEX \"idx_item\" ON " + JOIN_ITEM_TABLE_FULL_NAME + " (name) INCLUDE (price, discount1, discount2, \"supplier_id\", description)", - "CREATE INDEX \"idx_supplier\" ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + " (name)" }, - { "SORT-MERGE-JOIN (LEFT) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_supplier\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"S.:supplier_id\"]\n" + " CLIENT MERGE SORT\n" + "AND\n" - + " SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_item\n" + " SERVER SORTED BY [\"I.:item_id\"]\n" - + " CLIENT MERGE SORT\n" + " AND (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER FILTER BY QUANTITY < 5000\n" - + " SERVER SORTED BY [\"O.item_id\"]\n" + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY [\"I.0:supplier_id\"]", - - "SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_item\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I.:item_id\"]\n" + " CLIENT MERGE SORT\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER SORTED BY [\"O.item_id\"]\n" + " CLIENT MERGE SORT\n" - + "CLIENT 4 ROW LIMIT", - - "SORT-MERGE-JOIN (INNER) TABLES\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER Join.idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I1.:item_id\"]\n" + " CLIENT MERGE SORT\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER Join.idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I2.:item_id\"]\n" + " CLIENT MERGE SORT" } }); + "CREATE INDEX \"idx_supplier\" ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + " (name)" } }); return testCases; } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinIT.java index 014a0a8d496..ff65bdcbd67 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinIT.java @@ -40,7 +40,6 @@ import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -48,10 +47,58 @@ @RunWith(Parameterized.class) public abstract class SortMergeJoinIT extends BaseJoinIT { - public SortMergeJoinIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public SortMergeJoinIT(String[] indexDDL) { + super(indexDDL); } + /* + * The expected EXPLAIN plan for each of the queries below differs per index configuration, so + * each concrete subclass supplies the attribute-based assertions via these hooks. + */ + + /** + * {@link #testJoinWithSkipMergeOptimization()}: + * + * 
+   * SELECT /*+ USE_SORT_MERGE_JOIN*/ s.name FROM joinItemTable i
+   *   JOIN joinOrderTable o ON o.item_id = i.item_id AND quantity < 5000
+   *   RIGHT JOIN joinSupplierTable s ON i.supplier_id = s.supplier_id
+   *
+ */ + protected abstract void assertSkipMergeOptimizationPlan(Connection conn, String query) + throws Exception; + + /** + * {@link #testSelfJoin()}: + * + * 
+   * SELECT /*+ USE_SORT_MERGE_JOIN*/ i2.item_id, i1.name FROM joinItemTable i1
+   *   JOIN joinItemTable i2 ON i1.item_id = i2.item_id
+   *   ORDER BY i1.item_id
+   *
+ */ + protected abstract void assertSelfJoinPlan(Connection conn, String query) throws Exception; + + /** + * Assert the EXPLAIN plan for {@link #testJoinWithSetMaxRows()} (with a max-rows limit of 4). The + * {@code CLIENT 4 ROW LIMIT} comes from {@link java.sql.Statement#setMaxRows(int)} rather than + * the SQL, so subclasses must compile via a {@code PhoenixPreparedStatement}. + * + * 
+   * statement.setMaxRows(4);
+   *
+   * // queryIndex 0:
+   * SELECT /*+ USE_SORT_MERGE_JOIN*/ order_id, i.name, quantity FROM joinItemTable i
+   *   JOIN joinOrderTable o ON o.item_id = i.item_id
+   *
+   * // queryIndex 1:
+   * SELECT /*+ USE_SORT_MERGE_JOIN*/ o.order_id, i.name, o.quantity FROM joinItemTable i
+   *   JOIN (SELECT order_id, item_id, quantity FROM joinOrderTable) o ON o.item_id = i.item_id
+   *
+ */ + protected abstract void assertSetMaxRowsPlan(Connection conn, String query, int queryIndex) + throws Exception; + @Test public void testDefaultJoin() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -1663,8 +1710,7 @@ public void testJoinWithSkipMergeOptimization() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[0], QueryUtil.getExplainPlan(rs)); + assertSkipMergeOptimizationPlan(conn, query); } finally { conn.close(); } @@ -1707,8 +1753,7 @@ public void testSelfJoin() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query1); - assertPlansEqual(plans[2], QueryUtil.getExplainPlan(rs)); + assertSelfJoinPlan(conn, query1); statement = conn.prepareStatement(query2); rs = statement.executeQuery(); @@ -2581,9 +2626,7 @@ public void testJoinWithSetMaxRows() throws Exception { assertFalse(rs.next()); - rs = statement.executeQuery("EXPLAIN " + query); - assertPlansEqual(i == 0 ? plans[1] : plans[1].replaceFirst("O\\.item_id", "item_id"), - QueryUtil.getExplainPlan(rs)); + assertSetMaxRowsPlan(conn, query, i); } } finally { conn.close(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinLocalIndexIT.java index 8b03fdc7729..7a2b60ad0d0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinLocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinLocalIndexIT.java @@ -17,10 +17,16 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; + +import java.sql.Connection; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.util.SchemaUtil; import org.junit.experimental.categories.Category; import org.junit.runners.Parameterized.Parameters; @@ -45,48 +51,71 @@ protected Map getTableNameMap() { return virtualNameToRealNameMap; } - public SortMergeJoinLocalIndexIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public SortMergeJoinLocalIndexIT(String[] indexDDL) { + super(indexDDL); + } + + @Override + protected void assertSkipMergeOptimizationPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + String supplierIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_SUPPLIER_INDEX); + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (LEFT)").sortMergeSkipMerge(false) + .lhs().scanType("RANGE SCAN").table(supplierIndex + "(" + supplier + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).serverSortedBy("[\"S.:supplier_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end().rhs() + .abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(true) + .clientSortedBy("[\"I.0:supplier_id\"]").lhs().scanType("RANGE SCAN") + .table(itemIndex + "(" + item + ")").keyRanges("[1]").serverSortedBy("[\"I.:item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end().rhs().scanType("FULL SCAN").table(order) + .serverWhereFilter("SERVER FILTER BY QUANTITY < 5000").serverSortedBy("[\"O.item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end().end(); + } + + @Override + protected void assertSelfJoinPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(false) + .lhs().scanType("RANGE SCAN").table(itemIndex + "(" + item + ")").keyRanges("[1]") + .serverFirstKeyOnlyProjection(true).serverSortedBy("[\"I1.:item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end().rhs().scanType("RANGE SCAN") + .table(itemIndex + "(" + item + ")").keyRanges("[1]").serverFirstKeyOnlyProjection(true) + .serverSortedBy("[\"I2.:item_id\"]").clientSortAlgo("CLIENT MERGE SORT").end(); + } + + @Override + protected void assertSetMaxRowsPlan(Connection conn, String query, int queryIndex) + throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + String itemIndex = SchemaUtil.getTableName(getSchemaName(), JOIN_ITEM_INDEX); + PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class); + statement.setMaxRows(4); + ExplainPlanAttributes attributes = + statement.optimizeQuery().getExplainPlan().getPlanStepsAsAttributes(); + assertPlan(attributes).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(false) + .clientRowLimit(4).lhs().scanType("RANGE SCAN").table(itemIndex + "(" + item + ")") + .keyRanges("[1]").serverFirstKeyOnlyProjection(true).serverSortedBy("[\"I.:item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end().rhs().scanType("FULL SCAN").table(order) + .serverSortedBy(queryIndex == 0 ? "[\"O.item_id\"]" : "[\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end(); } - @Parameters(name = "SortMergeJoinLocalIndexIT_{index}") // name is used by failsafe as file name - // in reports + // name is used by failsafe as file name in reports + @Parameters(name = "SortMergeJoinLocalIndexIT_{index}") public static synchronized Collection data() { List testCases = Lists.newArrayList(); - testCases.add(new String[][] { - { "CREATE LOCAL INDEX " + JOIN_CUSTOMER_INDEX + " ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + testCases.add(new String[][] { { + "CREATE LOCAL INDEX " + JOIN_CUSTOMER_INDEX + " ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + "(name)", - "CREATE LOCAL INDEX " + JOIN_ITEM_INDEX + " ON " + JOIN_ITEM_TABLE_FULL_NAME + "(name) " - + "INCLUDE (price, discount1, discount2, \"supplier_id\", description)", - "CREATE LOCAL INDEX " + JOIN_SUPPLIER_INDEX + " ON " + JOIN_SUPPLIER_TABLE_FULL_NAME - + " (name)" }, - { "SORT-MERGE-JOIN (LEFT) TABLES\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" + JOIN_SUPPLIER_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"S.:supplier_id\"]\n" + " CLIENT MERGE SORT\n" + "AND\n" - + " SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER SORTED BY [\"I.:item_id\"]\n" + " CLIENT MERGE SORT\n" - + " AND (SKIP MERGE)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" + " SERVER FILTER BY QUANTITY < 5000\n" - + " SERVER SORTED BY [\"O.item_id\"]\n" + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY [\"I.0:supplier_id\"]", - - "SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I.:item_id\"]\n" + " CLIENT MERGE SORT\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER SORTED BY [\"O.item_id\"]\n" + " CLIENT MERGE SORT\n" - + "CLIENT 4 ROW LIMIT", - - "SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I1.:item_id\"]\n" + " CLIENT MERGE SORT\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I2.:item_id\"]\n" + " CLIENT MERGE SORT" } }); + "CREATE LOCAL INDEX " + JOIN_ITEM_INDEX + " ON " + JOIN_ITEM_TABLE_FULL_NAME + "(name) " + + "INCLUDE (price, discount1, discount2, \"supplier_id\", description)", + "CREATE LOCAL INDEX " + JOIN_SUPPLIER_INDEX + " ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + + " (name)" } }); return testCases; } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinNoIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinNoIndexIT.java index 73def45f5d9..20c7fe620c7 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinNoIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinNoIndexIT.java @@ -17,10 +17,15 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; + +import java.sql.Connection; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.junit.experimental.categories.Category; import org.junit.runners.Parameterized.Parameters; @@ -46,31 +51,52 @@ protected Map getTableNameMap() { return virtualNameToRealNameMap; } - public SortMergeJoinNoIndexIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public SortMergeJoinNoIndexIT(String[] indexDDL) { + super(indexDDL); } - @Parameters(name = "SortMergeJoinNoIndexIT_{index}") // name is used by failsafe as file name in - // reports - public static synchronized Collection data() { - List testCases = Lists.newArrayList(); - testCases.add(new String[][] { {}, - { "SORT-MERGE-JOIN (LEFT) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" + "AND\n" + " SORT-MERGE-JOIN (INNER) TABLES\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " AND (SKIP MERGE)\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" + " SERVER FILTER BY QUANTITY < 5000\n" - + " SERVER SORTED BY [\"O.item_id\"]\n" + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY [\"I.supplier_id\"]", + @Override + protected void assertSkipMergeOptimizationPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String supplier = getTableName(conn, JOIN_SUPPLIER_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (LEFT)").sortMergeSkipMerge(false) + .lhs().scanType("FULL SCAN").table(supplier).end().rhs() + .abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(true) + .clientSortedBy("[\"I.supplier_id\"]").lhs().scanType("FULL SCAN").table(item).end().rhs() + .scanType("FULL SCAN").table(order).serverWhereFilter("SERVER FILTER BY QUANTITY < 5000") + .serverSortedBy("[\"O.item_id\"]").clientSortAlgo("CLIENT MERGE SORT").end().end(); + } + + @Override + protected void assertSelfJoinPlan(Connection conn, String query) throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(false) + .lhs().scanType("FULL SCAN").table(item).end().rhs().scanType("FULL SCAN").table(item) + .serverFirstKeyOnlyProjection(true).end(); + } - "SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" + "AND\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + "\n" + " SERVER SORTED BY [\"O.item_id\"]\n" - + " CLIENT MERGE SORT\n" + "CLIENT 4 ROW LIMIT", + @Override + protected void assertSetMaxRowsPlan(Connection conn, String query, int queryIndex) + throws Exception { + String item = getTableName(conn, JOIN_ITEM_TABLE_FULL_NAME); + String order = getTableName(conn, JOIN_ORDER_TABLE_FULL_NAME); + PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class); + statement.setMaxRows(4); + ExplainPlanAttributes attributes = + statement.optimizeQuery().getExplainPlan().getPlanStepsAsAttributes(); + assertPlan(attributes).abstractExplainPlan("SORT-MERGE-JOIN (INNER)").sortMergeSkipMerge(false) + .clientRowLimit(4).lhs().scanType("FULL SCAN").table(item).end().rhs().scanType("FULL SCAN") + .table(order).serverSortedBy(queryIndex == 0 ? "[\"O.item_id\"]" : "[\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end(); + } - "SORT-MERGE-JOIN (INNER) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" + "AND\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" + " SERVER FILTER BY FIRST KEY ONLY" } }); + // name is used by failsafe as file name in reports + @Parameters(name = "SortMergeJoinNoIndexIT_{index}") + public static synchronized Collection data() { + List testCases = Lists.newArrayList(); + testCases.add(new String[][] { {} }); return testCases; } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinNoSpoolingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinNoSpoolingIT.java index 8f5e453c9cc..c5efcc4eb0b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinNoSpoolingIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SortMergeJoinNoSpoolingIT.java @@ -43,8 +43,8 @@ @Category(NeedsOwnMiniClusterTest.class) public class SortMergeJoinNoSpoolingIT extends SortMergeJoinNoIndexIT { - public SortMergeJoinNoSpoolingIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public SortMergeJoinNoSpoolingIT(String[] indexDDL) { + super(indexDDL); } @Parameters(name = "SortMergeJoinNoSpoolingIT_{index}") // name is used by failsafe as file name diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java index 4234253cd98..a21102f7419 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -35,7 +36,6 @@ import java.util.Properties; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -47,211 +47,26 @@ @Category(ParallelStatsDisabledTest.class) @RunWith(Parameterized.class) public class SubqueryIT extends BaseJoinIT { - public SubqueryIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public SubqueryIT(String[] indexDDL) { + super(indexDDL); } @Parameters public static synchronized Collection data() { List testCases = Lists.newArrayList(); - testCases.add(new String[][] { {}, - { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" + " SERVER SORTED BY \\[I.NAME\\]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SKIP-SCAN-JOIN TABLE 1\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + " \\['000000000000001'\\] - \\[\\*\\]\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I.item_id\" IN \\(\\$\\d+.\\$\\d+\\)", - - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" - + " SERVER SORTED BY [I.NAME]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " PARALLEL SEMI-JOIN TABLE 1(DELAYED EVALUATION) (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - - "CLIENT PARALLEL 4-WAY FULL SCAN OVER " + JOIN_COITEM_TABLE_FULL_NAME + "\n" - + "CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\".+.item_id\", .+.NAME\\]\n" - + " CLIENT MERGE SORT\n" - + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\".+.item_id\", .+.NAME\\]\n" - + " CLIENT MERGE SORT\n" + " SKIP-SCAN-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " DYNAMIC SERVER FILTER BY \"" - + JOIN_ITEM_TABLE_FULL_NAME + ".item_id\" IN \\(\\$\\d+.\\$\\d+\\)\n" - + " AFTER-JOIN SERVER FILTER BY \\(\\$\\d+.\\$\\d+ IS NOT NULL OR \\$\\d+.\\$\\d+ IS NOT NULL\\)", - - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER SORTED BY [I.NAME]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL ANTI-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_CUSTOMER_TABLE_FULL_NAME + "\n" - + " SKIP-SCAN-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I.item_id\" IN \\(\"O.item_id\"\\)\n" - + " AFTER-JOIN SERVER FILTER BY \\(I.NAME = 'T2' OR O.QUANTITY > \\$\\d+.\\$\\d+\\)\n" - + " DYNAMIC SERVER FILTER BY \"" + JOIN_CUSTOMER_TABLE_FULL_NAME - + ".customer_id\" IN \\(\\$\\d+.\\$\\d+\\)" } }); + testCases.add(new String[][] { {} }); testCases.add(new String[][] { { "CREATE INDEX \"idx_customer\" ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + " (name)", "CREATE INDEX \"idx_item\" ON " + JOIN_ITEM_TABLE_FULL_NAME + " (name) INCLUDE (price, discount1, discount2, \"supplier_id\", description)", - "CREATE INDEX \"idx_supplier\" ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + " (name)" }, - { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_supplier\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " PARALLEL SEMI-JOIN TABLE 1 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + " \\['000000000000001'\\] - \\[\\*\\]\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT", - - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_supplier\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER SORTED BY [\"I.0:NAME\"]\n" - + "CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " PARALLEL SEMI-JOIN TABLE 1(DELAYED EVALUATION) (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - - "CLIENT PARALLEL 4-WAY FULL SCAN OVER " + JOIN_COITEM_TABLE_FULL_NAME + "\n" - + "CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" - + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" - + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " AFTER-JOIN SERVER FILTER BY \\(\\$\\d+.\\$\\d+ IS NOT NULL OR \\$\\d+.\\$\\d+ IS NOT NULL\\)", - - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " PARALLEL ANTI-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - - "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_customer\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " AFTER-JOIN SERVER FILTER BY \\(\"I.0:NAME\" = 'T2' OR O.QUANTITY > \\$\\d+.\\$\\d+\\)" } }); - testCases.add(new String[][] { - { "CREATE LOCAL INDEX " + JOIN_CUSTOMER_INDEX + " ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + "CREATE INDEX \"idx_supplier\" ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + " (name)" } }); + testCases.add(new String[][] { { + "CREATE LOCAL INDEX " + JOIN_CUSTOMER_INDEX + " ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + " (name)", - "CREATE LOCAL INDEX " + JOIN_ITEM_INDEX + " ON " + JOIN_ITEM_TABLE_FULL_NAME + " " - + "(name) INCLUDE (price, discount1, discount2, \"supplier_id\", description)", - "CREATE LOCAL INDEX " + JOIN_SUPPLIER_INDEX + " ON " + JOIN_SUPPLIER_TABLE_FULL_NAME - + " (name)" }, - { "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "\\(" - + JOIN_ITEM_TABLE_FULL_NAME + "\\) \\[1\\]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL INNER-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_SUPPLIER_INDEX_FULL_NAME + "\\(" + JOIN_SUPPLIER_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " CLIENT MERGE SORT\n" - + " PARALLEL SEMI-JOIN TABLE 1 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + " \\['000000000000001'\\] - \\[\\*\\]\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I.:item_id\" IN \\(\\$\\d+.\\$\\d+\\)", - - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" - + JOIN_SUPPLIER_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"I.0:NAME\"]\n" + "CLIENT MERGE SORT\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "(" + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " CLIENT MERGE SORT\n" - + " PARALLEL SEMI-JOIN TABLE 1(DELAYED EVALUATION) (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - - "CLIENT PARALLEL 4-WAY FULL SCAN OVER " + JOIN_COITEM_TABLE_FULL_NAME + "\n" - + "CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "\\(" - + JOIN_ITEM_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL LEFT-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "\\(" - + JOIN_ITEM_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " DYNAMIC SERVER FILTER BY \"" - + JOIN_ITEM_INDEX_FULL_NAME + ".:item_id\" IN \\(\\$\\d+.\\$\\d+\\)\n" - + " AFTER-JOIN SERVER FILTER BY \\(\\$\\d+.\\$\\d+ IS NOT NULL OR \\$\\d+.\\$\\d+ IS NOT NULL\\)", - - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + "CLIENT MERGE SORT\n" + " PARALLEL ANTI-JOIN TABLE 0 (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT", - - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_CUSTOMER_INDEX_FULL_NAME + "\\(" - + JOIN_CUSTOMER_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT\n" - + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "\\(" - + JOIN_ITEM_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I.:item_id\" IN \\(\"O.item_id\"\\)\n" - + " AFTER-JOIN SERVER FILTER BY \\(\"I.0:NAME\" = 'T2' OR O.QUANTITY > \\$\\d+.\\$\\d+\\)\n" - + " DYNAMIC SERVER FILTER BY \"" + JOIN_CUSTOMER_INDEX_FULL_NAME - + ".:customer_id\" IN " + "\\(\\$\\d+.\\$\\d+\\)" } }); + "CREATE LOCAL INDEX " + JOIN_ITEM_INDEX + " ON " + JOIN_ITEM_TABLE_FULL_NAME + " " + + "(name) INCLUDE (price, discount1, discount2, \"supplier_id\", description)", + "CREATE LOCAL INDEX " + JOIN_SUPPLIER_INDEX + " ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + + " (name)" } }); return testCases; } @@ -433,9 +248,10 @@ public void testInSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String plan = QueryUtil.getExplainPlan(rs); - assertPlansMatch(plans[0], plan); + assertPlan(conn, query).subPlan(1).scanType("RANGE SCAN").table(tableName4) + .keyRanges("['000000000000001'] - [*]") + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); query = "SELECT i.\"item_id\", s.name FROM " + tableName2 + " s LEFT JOIN " + tableName1 + " i ON i.\"supplier_id\" = s.\"supplier_id\" WHERE i.\"item_id\" IN (SELECT \"item_id\" FROM " @@ -457,8 +273,9 @@ public void testInSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[1], QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).subPlan(1).scanType("FULL SCAN").table(tableName4) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); query = "SELECT * FROM " + tableName5 + " WHERE (item_id, item_name) IN (SELECT \"item_id\", name FROM " + tableName1 @@ -480,9 +297,7 @@ public void testInSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - plan = QueryUtil.getExplainPlan(rs); - assertPlansMatch(plans[2], plan); + assertCoitemDoubleSubqueryPlan(conn, query, tableName4, tableName5); } finally { conn.close(); } @@ -513,8 +328,9 @@ public void testExistsSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[3], QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).subPlan(0).scanType("FULL SCAN").table(tableName4) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); query = "SELECT * FROM " + tableName5 + " co WHERE EXISTS (SELECT 1 FROM " + tableName1 + " i WHERE NOT EXISTS (SELECT 1 FROM " + tableName4 @@ -537,9 +353,7 @@ public void testExistsSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String plan = QueryUtil.getExplainPlan(rs); - assertPlansMatch(plans[2], plan); + assertCoitemDoubleSubqueryPlan(conn, query, tableName4, tableName5); // PHOENIX-3633 query = "SELECT * FROM " + tableName4 + " o WHERE NOT EXISTS (SELECT 1 FROM " + tableName1 @@ -627,9 +441,11 @@ public void testComparisonSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String plan = QueryUtil.getExplainPlan(rs); - assertPlansMatch(plans[4], plan); + assertPlan(conn, query).subPlan(0) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"O.customer_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").subPlan(1).scanType("FULL SCAN").table(tableName4) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); query = "SELECT \"order_id\" FROM " + tableName4 + " o WHERE quantity = (SELECT quantity FROM " + tableName4 @@ -1121,4 +937,15 @@ public void testSubqueryReturnSingleAndCompare() throws Exception { assertEquals("C4", rs2.getString("NAME")); } } + + private void assertCoitemDoubleSubqueryPlan(Connection conn, String query, String orderTable, + String coitemTable) throws SQLException { + assertPlan(conn, query).scanType("FULL SCAN").table(coitemTable).iteratorType("PARALLEL") + .clientSortAlgo("CLIENT MERGE SORT").subPlanCount(2).subPlan(0).subPlan(0) + .scanType("FULL SCAN").table(orderTable) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT").end().end().subPlan(1).subPlan(0).scanType("FULL SCAN") + .table(orderTable).serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); + } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java index 81f79d78cd8..f780fcc3e14 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/join/SubqueryUsingSortMergeJoinIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.join; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -39,7 +40,6 @@ import org.apache.phoenix.execute.TupleProjectionPlan; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -52,173 +52,26 @@ @RunWith(Parameterized.class) public class SubqueryUsingSortMergeJoinIT extends BaseJoinIT { - public SubqueryUsingSortMergeJoinIT(String[] indexDDL, String[] plans) { - super(indexDDL, plans); + public SubqueryUsingSortMergeJoinIT(String[] indexDDL) { + super(indexDDL); } @Parameters public static synchronized Collection data() { List testCases = Lists.newArrayList(); - testCases.add(new String[][] { {}, { - "SORT-MERGE-JOIN (SEMI) TABLES\n" + " SORT-MERGE-JOIN (INNER) TABLES\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER SORTED BY [\"I.supplier_id\"]\n" + " CLIENT MERGE SORT\n" - + " AND\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SUPPLIER_TABLE_FULL_NAME + "\n" + " CLIENT SORTED BY [\"I.item_id\"]\n" - + "AND (SKIP MERGE)\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ORDER_TABLE_FULL_NAME + " ['000000000000001'] - [*]\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT\n" + "CLIENT SORTED BY [I.NAME]", - - "SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" + " SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" - + " CLIENT PARALLEL 4-WAY FULL SCAN OVER " + JOIN_COITEM_TABLE_FULL_NAME + "\n" - + " CLIENT MERGE SORT\n" + " AND\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\".+.item_id\", .+.NAME\\]\n" - + " CLIENT MERGE SORT\n" - + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"]\\\n" - + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY \\[.*.CO_ITEM_ID, .*.CO_ITEM_NAME\\]\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\".+.item_id\", .+.NAME\\]\n" - + " CLIENT MERGE SORT\n" + " SKIP-SCAN-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " DYNAMIC SERVER FILTER BY \"" - + JOIN_ITEM_TABLE_FULL_NAME + ".item_id\" IN \\(\\$\\d+.\\$\\d+\\)\n" - + "CLIENT FILTER BY \\(\\$\\d+.\\$\\d+ IS NOT NULL OR \\$\\d+.\\$\\d+ IS NOT NULL\\)", - - "SORT-MERGE-JOIN \\(SEMI\\) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_CUSTOMER_TABLE_FULL_NAME + "\n" + "AND \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I.item_id\" IN \\(\"O.item_id\"\\)\n" - + " AFTER-JOIN SERVER FILTER BY \\(I.NAME = 'T2' OR O.QUANTITY > \\$\\d+.\\$\\d+\\)", } }); + testCases.add(new String[][] { {} }); testCases.add(new String[][] { { "CREATE INDEX \"idx_customer\" ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + " (name)", "CREATE INDEX \"idx_item\" ON " + JOIN_ITEM_TABLE_FULL_NAME + " (name) INCLUDE (price, discount1, discount2, \"supplier_id\", description)", - "CREATE INDEX \"idx_supplier\" ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + " (name)" }, - { "SORT-MERGE-JOIN (SEMI) TABLES\n" + " SORT-MERGE-JOIN (INNER) TABLES\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER SORTED BY [\"I.0:supplier_id\"]\n" + " CLIENT MERGE SORT\n" - + " AND\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA - + ".idx_supplier\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"S.:supplier_id\"]\n" + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY [\"I.:item_id\"]\n" + "AND (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + " ['000000000000001'] - [*]\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT\n" + "CLIENT SORTED BY [\"I.0:NAME\"]", - - "SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" + " SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" - + " CLIENT PARALLEL 4-WAY FULL SCAN OVER " + JOIN_COITEM_TABLE_FULL_NAME + "\n" - + " CLIENT MERGE SORT\n" + " AND\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" - + " CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" - + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY \\[.*.CO_ITEM_ID, .*.CO_ITEM_NAME\\]\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" - + " CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" - + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + "CLIENT FILTER BY \\(\\$\\d+.\\$\\d+ IS NOT NULL OR \\$\\d+.\\$\\d+ IS NOT NULL\\)", - - "SORT-MERGE-JOIN \\(SEMI\\) TABLES\n" + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " - + JOIN_SCHEMA + ".idx_customer\n" + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY \\[\"Join.idx_customer.:customer_id\"\\]\n" - + " CLIENT MERGE SORT\n" + "AND \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SCHEMA + ".idx_item\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " AFTER-JOIN SERVER FILTER BY \\(\"I.0:NAME\" = 'T2' OR O.QUANTITY > \\$\\d+.\\$\\d+\\)", } }); - testCases.add(new String[][] { - { "CREATE LOCAL INDEX " + JOIN_CUSTOMER_INDEX + " ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + "CREATE INDEX \"idx_supplier\" ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + " (name)" } }); + testCases.add(new String[][] { { + "CREATE LOCAL INDEX " + JOIN_CUSTOMER_INDEX + " ON " + JOIN_CUSTOMER_TABLE_FULL_NAME + " (name)", - "CREATE LOCAL INDEX " + JOIN_ITEM_INDEX + " ON " + JOIN_ITEM_TABLE_FULL_NAME - + " (name) INCLUDE (price, discount1, discount2, \"supplier_id\", description)", - "CREATE LOCAL INDEX " + JOIN_SUPPLIER_INDEX + " ON " + JOIN_SUPPLIER_TABLE_FULL_NAME - + " (name)" }, - { "SORT-MERGE-JOIN (SEMI) TABLES\n" + " SORT-MERGE-JOIN (INNER) TABLES\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "(" - + JOIN_ITEM_TABLE_FULL_NAME + ") [1]\n" - + " SERVER SORTED BY [\"I.0:supplier_id\"]\n" + " CLIENT MERGE SORT\n" - + " AND\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_SUPPLIER_INDEX_FULL_NAME + "(" + JOIN_SUPPLIER_TABLE_FULL_NAME + ") [1]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER SORTED BY [\"S.:supplier_id\"]\n" + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY [\"I.:item_id\"]\n" + "AND (SKIP MERGE)\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + " ['000000000000001'] - [*]\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]\n" - + " CLIENT MERGE SORT\n" + "CLIENT SORTED BY [\"I.0:NAME\"]", - - "SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" + " SORT-MERGE-JOIN \\(LEFT\\) TABLES\n" - + " CLIENT PARALLEL 4-WAY FULL SCAN OVER " + JOIN_COITEM_TABLE_FULL_NAME + "\n" - + " CLIENT MERGE SORT\n" + " AND\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "\\(" - + JOIN_ITEM_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" - + " PARALLEL ANTI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME - + "\n" + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY \\[.*.CO_ITEM_ID, .*.CO_ITEM_NAME\\]\n" + "AND\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + JOIN_ITEM_INDEX_FULL_NAME + "\\(" - + JOIN_ITEM_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY \\[\".+.0:NAME\", \".+.:item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " CLIENT SORTED BY \\[\".+.:item_id\", \".+.0:NAME\"\\]\n" - + " PARALLEL SEMI-JOIN TABLE 0 \\(SKIP MERGE\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " DYNAMIC SERVER FILTER BY \"" - + JOIN_ITEM_INDEX_FULL_NAME + ".:item_id\" IN \\(\\$\\d+" + ".\\$\\d+\\)\n" - + "CLIENT FILTER BY \\(\\$\\d+.\\$\\d+ IS NOT NULL OR \\$\\d+.\\$\\d+ IS NOT NULL\\)", - - "SORT-MERGE-JOIN \\(SEMI\\) TABLES\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_CUSTOMER_INDEX_FULL_NAME + "\\(" + JOIN_CUSTOMER_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER SORTED BY \\[\"" - + JOIN_CUSTOMER_INDEX_FULL_NAME + ".:customer_id\"\\]\n" + " CLIENT MERGE SORT\n" - + "AND \\(SKIP MERGE\\)\n" + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " - + JOIN_ITEM_INDEX_FULL_NAME + "\\(" + JOIN_ITEM_TABLE_FULL_NAME + "\\) \\[1\\]\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"O.customer_id\"\\]\n" - + " CLIENT MERGE SORT\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " PARALLEL LEFT-JOIN TABLE 1\\(DELAYED EVALUATION\\)\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_FULL_NAME + "\n" - + " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[\"item_id\"\\]\n" - + " CLIENT MERGE SORT\n" - + " DYNAMIC SERVER FILTER BY \"I.:item_id\" IN \\(\"O.item_id\"\\)\n" - + " AFTER-JOIN SERVER FILTER BY \\(\"I.0:NAME\" = 'T2' OR O.QUANTITY > \\$\\d+.\\$\\d+\\)", } }); + "CREATE LOCAL INDEX " + JOIN_ITEM_INDEX + " ON " + JOIN_ITEM_TABLE_FULL_NAME + + " (name) INCLUDE (price, discount1, discount2, \"supplier_id\", description)", + "CREATE LOCAL INDEX " + JOIN_SUPPLIER_INDEX + " ON " + JOIN_SUPPLIER_TABLE_FULL_NAME + + " (name)" } }); return testCases; } @@ -284,8 +137,10 @@ public void testInSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertPlansEqual(plans[0], QueryUtil.getExplainPlan(rs)); + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (SEMI)").sortMergeSkipMerge(true) + .rhs().scanType("RANGE SCAN").table(tableName4).keyRanges("['000000000000001'] - [*]") + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); query = "SELECT /*+ USE_SORT_MERGE_JOIN*/ i.\"item_id\", s.name FROM " + tableName2 + " s LEFT JOIN " + tableName1 @@ -328,9 +183,11 @@ public void testInSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String plan = QueryUtil.getExplainPlan(rs); - assertPlansMatch(plans[1], plan); + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (LEFT)") + .sortMergeSkipMerge(false).lhs().lhs().scanType("FULL SCAN").table(tableName5) + .iteratorType("PARALLEL").end().end().rhs().subPlan(0).scanType("FULL SCAN") + .table(tableName4).serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); } finally { conn.close(); } @@ -383,9 +240,11 @@ public void testExistsSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String plan = QueryUtil.getExplainPlan(rs); - assertPlansMatch(plans[1], plan); + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (LEFT)") + .sortMergeSkipMerge(false).lhs().lhs().scanType("FULL SCAN").table(tableName5) + .iteratorType("PARALLEL").end().end().rhs().subPlan(0).scanType("FULL SCAN") + .table(tableName4).serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); } finally { conn.close(); } @@ -434,9 +293,10 @@ public void testComparisonSubquery() throws Exception { assertFalse(rs.next()); - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String plan = QueryUtil.getExplainPlan(rs); - assertPlansMatch(plans[2], plan); + assertPlan(conn, query).abstractExplainPlan("SORT-MERGE-JOIN (SEMI)").sortMergeSkipMerge(true) + .rhs().subPlan(1).scanType("FULL SCAN").table(tableName4) + .serverAggregate("SERVER AGGREGATE INTO DISTINCT ROWS BY [\"item_id\"]") + .clientSortAlgo("CLIENT MERGE SORT"); query = "SELECT /*+ USE_SORT_MERGE_JOIN*/ \"order_id\" FROM " + tableName4 + " o WHERE quantity = (SELECT quantity FROM " + tableName4 diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java index 5c79a6a0d37..ce5bdaf950e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.json; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -40,11 +41,11 @@ import java.util.Arrays; import java.util.Properties; import org.apache.commons.io.FileUtils; -import org.apache.phoenix.end2end.IndexToolIT; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.TableNotFoundException; @@ -110,9 +111,9 @@ public void testSimpleJsonValue() throws Exception { rs = conn.createStatement().executeQuery(query); assertFalse(rs.next()); - // check if the explain plan indicates server side execution + // Check here for the JSON server side projection rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertTrue(QueryUtil.getExplainPlan(rs).contains(" SERVER JSON FUNCTION PROJECTION")); + assertTrue(QueryUtil.getExplainPlan(rs).contains(" SERVER JSON PROJECTION ")); } } @@ -367,12 +368,11 @@ public void testJsonExpressionIndex() throws IOException { String selectSql = "SELECT JSON_VALUE(JSONCOL,'$.type'), " + "JSON_VALUE(JSONCOL,'$.info.address.town') FROM " + tableName + " WHERE JSON_VALUE(JSONCOL,'$.type') = 'Basic'"; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql); - String actualExplainPlan = QueryUtil.getExplainPlan(rs); - IndexToolIT.assertExplainPlan(false, actualExplainPlan, tableName, indexName); + assertPlan(conn, selectSql).scanType("RANGE SCAN").table(indexName) + .functionalMatch("JSON_VALUE(JSONCOL.JSONCOL,'$.type')").indexRejectedNone(); // Validate the total count of rows String countSql = "SELECT COUNT(1) FROM " + tableName; - rs = conn.createStatement().executeQuery(countSql); + ResultSet rs = conn.createStatement().executeQuery(countSql); assertTrue(rs.next()); assertEquals(5, rs.getInt(1)); // Delete the rows @@ -542,7 +542,7 @@ public void testJsonFunctionOptimization() throws Exception { String query = String.format(queryTemplate, "AndersenFamily"); // check if the explain plan indicates server side execution ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertFalse(QueryUtil.getExplainPlan(rs).contains(" SERVER JSON FUNCTION PROJECTION")); + assertFalse(QueryUtil.getExplainPlan(rs).contains(" SERVER JSON PROJECTION ")); } } @@ -568,8 +568,9 @@ public void testArrayIndexAndJsonFunctionExpressions() throws Exception { // Since we are using complete array and json col, no server side execution ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); String explainPlan = QueryUtil.getExplainPlan(rs); - assertFalse(explainPlan.contains(" SERVER JSON FUNCTION PROJECTION")); - assertFalse(explainPlan.contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertFalse(explainPlan.contains(" SERVER JSON PROJECTION ")); + assertPlan(conn, query).serverParsedProjectionsNone() + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(conn.createArrayOf("INTEGER", new Integer[] { 1, 2 }), rs.getArray(1)); @@ -582,24 +583,27 @@ public void testArrayIndexAndJsonFunctionExpressions() throws Exception { + " WHERE JSON_VALUE(jsoncol, '$.name') = 'AndersenFamily'"; rs = conn.createStatement().executeQuery("EXPLAIN " + query); explainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(explainPlan.contains(" SERVER JSON FUNCTION PROJECTION")); - assertTrue(explainPlan.contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertTrue(explainPlan.contains(" SERVER JSON PROJECTION ")); + assertPlan(conn, query).serverParsedProjectionCount("ARRAY", 1) + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); // only Array optimization and not Json query = "SELECT arr[1], jsoncol, JSON_VALUE(jsoncol, '$.type')" + " FROM " + tableName + " WHERE JSON_VALUE(jsoncol, '$.name') = 'AndersenFamily'"; rs = conn.createStatement().executeQuery("EXPLAIN " + query); explainPlan = QueryUtil.getExplainPlan(rs); - assertFalse(explainPlan.contains(" SERVER JSON FUNCTION PROJECTION")); - assertTrue(explainPlan.contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertFalse(explainPlan.contains(" SERVER JSON PROJECTION ")); + assertPlan(conn, query).serverParsedProjectionCount("ARRAY", 1) + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); // only Json optimization and not Array Index query = "SELECT arr, arr[1], JSON_VALUE(jsoncol, '$.type')" + " FROM " + tableName + " WHERE JSON_VALUE(jsoncol, '$.name') = 'AndersenFamily'"; rs = conn.createStatement().executeQuery("EXPLAIN " + query); explainPlan = QueryUtil.getExplainPlan(rs); - assertTrue(explainPlan.contains(" SERVER JSON FUNCTION PROJECTION")); - assertFalse(explainPlan.contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertTrue(explainPlan.contains(" SERVER JSON PROJECTION ")); + assertPlan(conn, query).serverParsedProjectionCount("ARRAY", 0) + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java index 24a46e02116..d75f443d355 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.salted; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TABLE_WITH_SALTING; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; @@ -31,7 +32,6 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryBuilder; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; @@ -183,7 +183,6 @@ public void testSelectValueWithFullyQualifiedWhereClause() throws Exception { Connection conn = DriverManager.getConnection(getUrl(), props); try { String tableName = initTableValues(null); - PreparedStatement stmt; ResultSet rs; // Variable length slot with bounded ranges. @@ -418,10 +417,9 @@ public void testSelectWithOrderByRowKey() throws Exception { String query = "SELECT * FROM " + tableName + " ORDER BY a_integer, a_string, a_id"; PreparedStatement statement = conn.prepareStatement(query); - ResultSet explainPlan = statement.executeQuery("EXPLAIN " + query); // Confirm that ORDER BY in row key order will be optimized out for salted table - assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + tableName + "\n" + "CLIENT MERGE SORT", - QueryUtil.getExplainPlan(explainPlan)); + assertPlan(conn, query).iteratorType("PARALLEL").scanType("FULL SCAN").table(tableName) + .clientSortAlgo("CLIENT MERGE SORT"); ResultSet rs = statement.executeQuery(); assertTrue(rs.next()); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java index 4794610b408..2c0760be947 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.end2end.salted; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -31,7 +32,6 @@ import java.util.Properties; import org.apache.phoenix.end2end.ParallelStatsDisabledTest; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -80,9 +80,7 @@ public void testPointLookupOnSaltedTable() throws Exception { ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals(1, rs.getInt("a_integer")); - query = "explain " + query; - rs = conn.createStatement().executeQuery(query); - assertTrue(QueryUtil.getExplainPlan(rs).contains("POINT LOOKUP ON 1 KEY")); + assertPlan(conn, query).scanType("POINT LOOKUP ON 1 KEY"); } } @@ -108,10 +106,8 @@ public void testPointLookupOnSaltedTable2() throws Exception { assertEquals(i, rs.getInt("A")); assertEquals(i + 10, rs.getInt("B")); assertFalse(rs.next()); - query = "explain " + query; - rs = conn.createStatement().executeQuery(query); - assertTrue(QueryUtil.getExplainPlan(rs) - .contains("CLIENT PARALLEL 1-WAY POINT LOOKUP ON 1 KEY OVER")); + assertPlan(conn, query).iteratorType("PARALLEL").scanType("POINT LOOKUP ON 1 KEY") + .table(tableName); } } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeAssertions.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeAssertions.java new file mode 100644 index 00000000000..e2a43291be4 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeAssertions.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.tpcds; + +import static org.junit.Assert.fail; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.phoenix.query.explain.ExplainPlanTestUtil; +import org.apache.phoenix.query.explain.ExplainPlanTestUtil.ExplainPlanAssert; + +/** + * Result and EXPLAIN assertion helpers for the TPC-DS derived ITs. + *

+ * {@link #assertResult(Connection, String, String, String[][])} compares the full, ordered result + * set of {@code sql} against an embedded {@code expected} array captured from the deterministic + * {@link TPCDSLikeFixture}. Each adapted query carries a total {@code ORDER BY} so the row order is + * stable. Cell values are compared via {@link ResultSet#getString(int)}. + *

+ * Regenerating expected arrays. Run either IT with + * {@code -D}{@value #REGENERATE_PROPERTY}{@code =true}. In that mode {@code assertResult} prints a + * paste-ready literal to stdout for each query. Paste the printed blocks back into the IT and + * commit. + */ +public final class TPCDSLikeAssertions { + + public static final String REGENERATE_PROPERTY = "phoenix.tpcds.regenerate"; + + public static final String REGENERATE_ENV = "PHOENIX_TPCDS_REGENERATE"; + + private static final boolean REGENERATE = Boolean.getBoolean(REGENERATE_PROPERTY) + || "true".equalsIgnoreCase(System.getenv(REGENERATE_ENV)); + + private TPCDSLikeAssertions() { + } + + public static boolean isRegenerating() { + return REGENERATE; + } + + /** + * Execute {@code sql} and either assert its rows equal {@code expected} (normal mode) or print a + * paste-ready expected-array literal (regenerate mode). + * @param conn a Phoenix connection + * @param label the query label, e.g. {@code "Q03"}; only used to name the printed literal + * @param sql the query to run + * @param expected the embedded expected rows (ignored in capture mode) + * @return {@code true} if the call captured/printed instead of asserting (regenerate mode, or an + * empty {@code expected} that has not yet been populated); {@code false} if it asserted + */ + public static boolean assertResult(Connection conn, String label, String sql, String[][] expected) + throws SQLException { + List actual = run(conn, sql); + // Capture mode: either forced via the flag, or bootstrapping an as-yet-unpopulated query. + if (REGENERATE || expected == null || expected.length == 0) { + System.out.println((expected == null || expected.length == 0 + ? "// NOTE: " + label + " has no embedded expected rows yet; paste the block below.\n" + : "") + format(label, actual)); + return true; + } + String mismatch = diff(expected, actual); + if (mismatch != null) { + fail("Result mismatch for " + label + ": " + mismatch + "\nRe-run with -D" + + REGENERATE_PROPERTY + "=true (or env " + REGENERATE_ENV + + "=true) to capture the current output.\nFull query:\n" + sql); + } + return false; + } + + public static String captureLiteral(Connection conn, String label, String sql) + throws SQLException { + return format(label, run(conn, sql)); + } + + /** Begin EXPLAIN-plan assertions for {@code sql} (delegates to {@link ExplainPlanTestUtil}). */ + public static ExplainPlanAssert assertPlan(Connection conn, String sql) throws SQLException { + return ExplainPlanTestUtil.assertPlan(conn, sql); + } + + /** Assert the rendered EXPLAIN plan text for {@code sql} contains every {@code needle}. */ + public static void assertPlanContains(Connection conn, String sql, String... needles) + throws SQLException { + List steps = ExplainPlanTestUtil.getPlanSteps(conn, sql); + String text = String.join("\n", steps); + for (String needle : needles) { + if (!text.contains(needle)) { + fail("EXPLAIN plan missing expected fragment:\n expected to contain: " + needle + + "\nactual plan:\n" + text + "\nquery:\n" + sql); + } + } + } + + /** + * Assert the rendered EXPLAIN plan text for {@code sql} contains at least one of {@code needles}. + */ + public static void assertPlanContainsAny(Connection conn, String sql, String... needles) + throws SQLException { + List steps = ExplainPlanTestUtil.getPlanSteps(conn, sql); + String text = String.join("\n", steps); + for (String needle : needles) { + if (text.contains(needle)) { + return; + } + } + fail("EXPLAIN plan missing all expected fragments " + Arrays.toString(needles) + ":\n" + text + + "\nquery:\n" + sql); + } + + private static List run(Connection conn, String sql) throws SQLException { + List rows = new ArrayList<>(); + try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery(sql)) { + ResultSetMetaData md = rs.getMetaData(); + int n = md.getColumnCount(); + while (rs.next()) { + String[] row = new String[n]; + for (int i = 1; i <= n; i++) { + row[i - 1] = canonical(rs.getObject(i)); + } + rows.add(row); + } + } + return rows; + } + + /** + * Canonicalize a JDBC value into a stable locale independent string. Numbers are normalized via + * {@link BigDecimal} with trailing zeros stripped (so {@code 37.00 -> "37"}, {@code 1081.50 -> + * "1081.5"}, {@code 0 -> "0"}). Everything else uses {@code toString()}. + */ + private static String canonical(Object value) { + if (value == null) { + return null; + } + if (value instanceof Number) { + BigDecimal b = new BigDecimal(value.toString()); + return b.signum() == 0 ? "0" : b.stripTrailingZeros().toPlainString(); + } + return value.toString(); + } + + private static String diff(String[][] expected, List actual) { + if (expected == null) { + expected = new String[0][]; + } + if (expected.length != actual.size()) { + return "row count expected=" + expected.length + " actual=" + actual.size(); + } + for (int r = 0; r < expected.length; r++) { + String[] e = expected[r]; + String[] a = actual.get(r); + if (e.length != a.length) { + return "row " + r + " column count expected=" + e.length + " actual=" + a.length; + } + for (int c = 0; c < e.length; c++) { + if (!eq(e[c], a[c])) { + return "row " + r + " col " + c + " expected=" + lit(e[c]) + " actual=" + lit(a[c]) + + "\n expectedRow=" + Arrays.toString(e) + "\n actualRow= " + Arrays.toString(a); + } + } + } + return null; + } + + private static boolean eq(String a, String b) { + return a == null ? b == null : a.equals(b); + } + + private static String lit(String s) { + return s == null ? "null" : "\"" + s + "\""; + } + + private static String format(String label, List rows) { + StringBuilder sb = new StringBuilder(); + sb.append("\n // ---- ").append(label).append(" (").append(rows.size()) + .append(" rows) ----\n"); + sb.append(" private static final String[][] ").append(label).append("_EXPECTED = {\n"); + for (String[] row : rows) { + sb.append(" { "); + for (int c = 0; c < row.length; c++) { + if (c > 0) { + sb.append(", "); + } + sb.append(lit(row[c])); + } + sb.append(" },\n"); + } + sb.append(" };\n"); + return sb.toString(); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeBaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeBaseIT.java new file mode 100644 index 00000000000..cdeb2310e64 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeBaseIT.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.tpcds; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Properties; +import org.apache.phoenix.end2end.ParallelStatsDisabledIT; +import org.apache.phoenix.util.PropertiesUtil; + +/** + * Common plumbing for the TPC-DS derived ITs. The suite is parameterized over the two schemas + * materialized by {@link TPCDSLikeFixture}. + *

+ * Adapted query constants are written against {@link TPCDSLikeFixture#SCHEMA}. + */ +public abstract class TPCDSLikeBaseIT extends ParallelStatsDisabledIT { + + protected final String schema; + protected final boolean noIndex; + + protected TPCDSLikeBaseIT(String label, String schema, boolean noIndex) { + this.schema = schema; + this.noIndex = noIndex; + } + + protected static Collection indexParameters() { + return Arrays.asList(new Object[][] { { "NO_INDEX", TPCDSLikeFixture.SCHEMA, true }, + { "GLOBAL_INDEX", TPCDSLikeFixture.SCHEMA_INDEXED, false } }); + } + + protected static void loadFixture() throws SQLException { + try (Connection conn = newConnection()) { + TPCDSLikeFixture.load(conn); + } + } + + protected static Connection newConnection() throws SQLException { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + return DriverManager.getConnection(getUrl(), props); + } + + /** Rewrite the canonical {@code TPCDS.} table prefix to the parameter's schema. */ + protected String sql(String base) { + if (TPCDSLikeFixture.SCHEMA.equals(schema)) { + return base; + } + return base.replace(TPCDSLikeFixture.SCHEMA + ".", schema + "."); + } + + private static final String[] NO_INDEXES = new String[0]; + + /** Assert a query's full result. */ + protected void check(String label, String baseSql, String[][] expected, String... markers) + throws SQLException { + check(label, baseSql, expected, markers, NO_INDEXES); + } + + /** + * As {@link #check(String, String, String[][], String...)}, but additionally asserts that the + * plan scans each covering index in {@code indexNames}. + */ + protected void check(String label, String baseSql, String[][] expected, String[] markers, + String[] indexNames) throws SQLException { + String resolved = sql(baseSql); + try (Connection conn = newConnection()) { + boolean captured = TPCDSLikeAssertions.assertResult(conn, label, resolved, expected); + if (captured) { + return; + } + if (markers.length > 0) { + TPCDSLikeAssertions.assertPlanContains(conn, resolved, markers); + } + if (!noIndex && indexNames.length > 0) { + String[] qualified = new String[indexNames.length]; + for (int i = 0; i < indexNames.length; i++) { + qualified[i] = schema + "." + indexNames[i]; + } + TPCDSLikeAssertions.assertPlanContains(conn, resolved, qualified); + } + } + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeCrossChannelIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeCrossChannelIT.java new file mode 100644 index 00000000000..7b32b98a91b --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeCrossChannelIT.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.tpcds; + +import java.sql.SQLException; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * TPC-DS derived integration tests, exercising unions and sort-merge joins. + *

+ * See {@link TPCDSLikeFixture} for the schema and the catalog of queries intentionally not ported, + * and {@link TPCDSLikeSingleChannelIT} for the single channel queries. + */ +@Category(ParallelStatsDisabledTest.class) +@RunWith(Parameterized.class) +public class TPCDSLikeCrossChannelIT extends TPCDSLikeBaseIT { + + public TPCDSLikeCrossChannelIT(String label, String schema, boolean noIndex) { + super(label, schema, noIndex); + } + + @Parameters(name = "{0}") + public static Collection params() { + return indexParameters(); + } + + @BeforeClass + public static synchronized void setup() throws SQLException { + loadFixture(); + } + + // TPC-DS Q56: total ext sales price by item across store/catalog/web for a color set and region. + static final String Q56 = "SELECT i_item_id, SUM(total_sales) agg_total_sales FROM (" + + " SELECT i.i_item_id i_item_id, SUM(ss.ss_ext_sales_price) total_sales" + + " FROM TPCDS.store_sales ss, TPCDS.date_dim d, TPCDS.customer_address ca, TPCDS.item i" + + " WHERE ss.ss_sold_date_sk = d.d_date_sk AND ss.ss_addr_sk = ca.ca_address_sk" + + " AND ss.ss_item_sk = i.i_item_sk AND i.i_color IN ('red', 'blue', 'green')" + + " AND d.d_year = 2000 AND ca.ca_gmt_offset = -5 GROUP BY i.i_item_id" + " UNION ALL" + + " SELECT i.i_item_id i_item_id, SUM(cs.cs_ext_sales_price) total_sales" + + " FROM TPCDS.catalog_sales cs, TPCDS.date_dim d, TPCDS.customer_address ca, TPCDS.item i" + + " WHERE cs.cs_sold_date_sk = d.d_date_sk AND cs.cs_ship_addr_sk = ca.ca_address_sk" + + " AND cs.cs_item_sk = i.i_item_sk AND i.i_color IN ('red', 'blue', 'green')" + + " AND d.d_year = 2000 AND ca.ca_gmt_offset = -5 GROUP BY i.i_item_id" + " UNION ALL" + + " SELECT i.i_item_id i_item_id, SUM(ws.ws_ext_sales_price) total_sales" + + " FROM TPCDS.web_sales ws, TPCDS.date_dim d, TPCDS.customer_address ca, TPCDS.item i" + + " WHERE ws.ws_sold_date_sk = d.d_date_sk AND ws.ws_ship_addr_sk = ca.ca_address_sk" + + " AND ws.ws_item_sk = i.i_item_sk AND i.i_color IN ('red', 'blue', 'green')" + + " AND d.d_year = 2000 AND ca.ca_gmt_offset = -5 GROUP BY i.i_item_id" + + " ) tmp GROUP BY i_item_id ORDER BY agg_total_sales, i_item_id LIMIT 100"; + + // TPC-DS Q60: cross-channel total ext sales price by item for a category set and region. + static final String Q60 = "SELECT i_item_id, SUM(total_sales) agg_total_sales FROM (" + + " SELECT i.i_item_id i_item_id, SUM(ss.ss_ext_sales_price) total_sales" + + " FROM TPCDS.store_sales ss, TPCDS.date_dim d, TPCDS.customer_address ca, TPCDS.item i" + + " WHERE ss.ss_sold_date_sk = d.d_date_sk AND ss.ss_addr_sk = ca.ca_address_sk" + + " AND ss.ss_item_sk = i.i_item_sk AND i.i_category IN ('Books', 'Music', 'Home')" + + " AND d.d_year = 2001 AND ca.ca_gmt_offset = -6 GROUP BY i.i_item_id" + " UNION ALL" + + " SELECT i.i_item_id i_item_id, SUM(cs.cs_ext_sales_price) total_sales" + + " FROM TPCDS.catalog_sales cs, TPCDS.date_dim d, TPCDS.customer_address ca, TPCDS.item i" + + " WHERE cs.cs_sold_date_sk = d.d_date_sk AND cs.cs_ship_addr_sk = ca.ca_address_sk" + + " AND cs.cs_item_sk = i.i_item_sk AND i.i_category IN ('Books', 'Music', 'Home')" + + " AND d.d_year = 2001 AND ca.ca_gmt_offset = -6 GROUP BY i.i_item_id" + " UNION ALL" + + " SELECT i.i_item_id i_item_id, SUM(ws.ws_ext_sales_price) total_sales" + + " FROM TPCDS.web_sales ws, TPCDS.date_dim d, TPCDS.customer_address ca, TPCDS.item i" + + " WHERE ws.ws_sold_date_sk = d.d_date_sk AND ws.ws_ship_addr_sk = ca.ca_address_sk" + + " AND ws.ws_item_sk = i.i_item_sk AND i.i_category IN ('Books', 'Music', 'Home')" + + " AND d.d_year = 2001 AND ca.ca_gmt_offset = -6 GROUP BY i.i_item_id" + + " ) tmp GROUP BY i_item_id ORDER BY agg_total_sales, i_item_id LIMIT 100"; + + // TPC-DS Q83: cross-channel return quantities by item. + static final String Q83 = "SELECT sr.item_id, sr.sr_item_qty, cr.cr_item_qty, wr.wr_item_qty" + + " FROM (SELECT i.i_item_id item_id, SUM(srt.sr_return_quantity) sr_item_qty" + + " FROM TPCDS.store_returns srt, TPCDS.item i, TPCDS.date_dim d" + + " WHERE srt.sr_item_sk = i.i_item_sk AND srt.sr_returned_date_sk = d.d_date_sk" + + " AND d.d_year IN (2000, 2001) GROUP BY i.i_item_id) sr" + + " JOIN (SELECT i.i_item_id item_id, SUM(crt.cr_return_quantity) cr_item_qty" + + " FROM TPCDS.catalog_returns crt, TPCDS.item i, TPCDS.date_dim d" + + " WHERE crt.cr_item_sk = i.i_item_sk AND crt.cr_returned_date_sk = d.d_date_sk" + + " AND d.d_year IN (2000, 2001) GROUP BY i.i_item_id) cr ON sr.item_id = cr.item_id" + + " JOIN (SELECT i.i_item_id item_id, SUM(wrt.wr_return_quantity) wr_item_qty" + + " FROM TPCDS.web_returns wrt, TPCDS.item i, TPCDS.date_dim d" + + " WHERE wrt.wr_item_sk = i.i_item_sk AND wrt.wr_returned_date_sk = d.d_date_sk" + + " AND d.d_year IN (2000, 2001) GROUP BY i.i_item_id) wr ON sr.item_id = wr.item_id" + + " ORDER BY sr.item_id LIMIT 100"; + + // TPC-DS Q97: store-only / catalog-only / both customer-item pairs via a FULL OUTER JOIN. + // The USE_SORT_MERGE_JOIN hint forces the sort-merge strategy Phoenix requires for FULL OUTER. + static final String Q97 = "SELECT /*+ USE_SORT_MERGE_JOIN */" + + " SUM(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NULL" + + " THEN 1 ELSE 0 END) store_only," + + " SUM(CASE WHEN ssci.customer_sk IS NULL AND csci.customer_sk IS NOT NULL" + + " THEN 1 ELSE 0 END) catalog_only," + + " SUM(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NOT NULL" + + " THEN 1 ELSE 0 END) store_and_catalog" + + " FROM (SELECT ss.ss_customer_sk customer_sk, ss.ss_item_sk item_sk" + + " FROM TPCDS.store_sales ss, TPCDS.date_dim d" + + " WHERE ss.ss_sold_date_sk = d.d_date_sk AND d.d_year = 2000" + + " GROUP BY ss.ss_customer_sk, ss.ss_item_sk) ssci" + + " FULL OUTER JOIN (SELECT cs.cs_bill_customer_sk customer_sk, cs.cs_item_sk item_sk" + + " FROM TPCDS.catalog_sales cs, TPCDS.date_dim d" + + " WHERE cs.cs_sold_date_sk = d.d_date_sk AND d.d_year = 2000" + + " GROUP BY cs.cs_bill_customer_sk, cs.cs_item_sk) csci" + + " ON ssci.customer_sk = csci.customer_sk AND ssci.item_sk = csci.item_sk LIMIT 100"; + + // TPC-DS Q88: counts of store sales in several time-of-day buckets, one per derived table, + // combined by cross join. + static final String Q88 = "SELECT * FROM" + + " (SELECT COUNT(*) h8 FROM TPCDS.store_sales ss, TPCDS.household_demographics hd," + + " TPCDS.time_dim t, TPCDS.store s" + + " WHERE ss.ss_sold_time_sk = t.t_time_sk AND ss.ss_hdemo_sk = hd.hd_demo_sk" + + " AND ss.ss_store_sk = s.s_store_sk AND t.t_hour = 8 AND hd.hd_dep_count >= 0" + + " AND s.s_company_id = 1) s1," + + " (SELECT COUNT(*) h10 FROM TPCDS.store_sales ss, TPCDS.household_demographics hd," + + " TPCDS.time_dim t, TPCDS.store s" + + " WHERE ss.ss_sold_time_sk = t.t_time_sk AND ss.ss_hdemo_sk = hd.hd_demo_sk" + + " AND ss.ss_store_sk = s.s_store_sk AND t.t_hour = 10 AND hd.hd_dep_count >= 0" + + " AND s.s_company_id = 1) s2," + + " (SELECT COUNT(*) h12 FROM TPCDS.store_sales ss, TPCDS.household_demographics hd," + + " TPCDS.time_dim t, TPCDS.store s" + + " WHERE ss.ss_sold_time_sk = t.t_time_sk AND ss.ss_hdemo_sk = hd.hd_demo_sk" + + " AND ss.ss_store_sk = s.s_store_sk AND t.t_hour = 12 AND hd.hd_dep_count >= 0" + + " AND s.s_company_id = 1) s3"; + + // TPC-DS Q1: customers whose store-return total exceeds 1.2x their store's average. + static final String Q01 = "SELECT c.c_customer_id" + + " FROM (SELECT sr.sr_customer_sk ctr_customer_sk, sr.sr_store_sk ctr_store_sk," + + " SUM(sr.sr_return_amt) ctr_total_return" + " FROM TPCDS.store_returns sr, TPCDS.date_dim d" + + " WHERE sr.sr_returned_date_sk = d.d_date_sk AND d.d_year = 2000" + + " GROUP BY sr.sr_customer_sk, sr.sr_store_sk) ctr1," + + " (SELECT x.ctr_store_sk ctr_store_sk, AVG(x.ctr_total_return) * 1.2 avg_return FROM" + + " (SELECT sr.sr_customer_sk ctr_customer_sk, sr.sr_store_sk ctr_store_sk," + + " SUM(sr.sr_return_amt) ctr_total_return" + " FROM TPCDS.store_returns sr, TPCDS.date_dim d" + + " WHERE sr.sr_returned_date_sk = d.d_date_sk AND d.d_year = 2000" + + " GROUP BY sr.sr_customer_sk, sr.sr_store_sk) x GROUP BY x.ctr_store_sk) ctr2," + + " TPCDS.store s, TPCDS.customer c" + + " WHERE ctr1.ctr_total_return > ctr2.avg_return AND s.s_store_sk = ctr1.ctr_store_sk" + + " AND ctr2.ctr_store_sk = ctr1.ctr_store_sk AND ctr1.ctr_customer_sk = c.c_customer_sk" + + " ORDER BY c.c_customer_id LIMIT 100"; + + private static final String[][] Q56_EXPECTED = + { { "ITEM0000000024", "47" }, { "ITEM0000000012", "237.5" }, { "ITEM0000000020", "758" }, + { "ITEM0000000002", "962" }, { "ITEM0000000018", "1081.5" }, { "ITEM0000000001", "1442.5" }, + { "ITEM0000000007", "1818" }, { "ITEM0000000008", "1957.5" }, { "ITEM0000000014", "2092" }, + { "ITEM0000000006", "2725" }, { "ITEM0000000013", "2854.5" }, { "ITEM0000000019", "5817" }, }; + private static final String[][] Q60_EXPECTED = { { "ITEM0000000001", "849" }, + { "ITEM0000000007", "897" }, { "ITEM0000000018", "1293.5" }, { "ITEM0000000013", "1650" }, + { "ITEM0000000019", "1784.5" }, { "ITEM0000000006", "2093.5" }, { "ITEM0000000024", "2345" }, + { "ITEM0000000020", "3031.5" }, { "ITEM0000000012", "3220" }, { "ITEM0000000002", "3226.5" }, + { "ITEM0000000014", "3287.5" }, { "ITEM0000000008", "3913.5" }, }; + private static final String[][] Q83_EXPECTED = + { { "ITEM0000000001", "17", "13", "11" }, { "ITEM0000000002", "20", "17", "10" }, + { "ITEM0000000003", "16", "12", "14" }, { "ITEM0000000004", "17", "12", "12" }, + { "ITEM0000000005", "13", "12", "13" }, { "ITEM0000000006", "18", "9", "7" }, + { "ITEM0000000007", "16", "15", "13" }, { "ITEM0000000008", "17", "9", "12" }, + { "ITEM0000000009", "23", "10", "12" }, { "ITEM0000000010", "16", "15", "17" }, + { "ITEM0000000011", "17", "13", "9" }, { "ITEM0000000012", "15", "14", "10" }, + { "ITEM0000000013", "12", "11", "5" }, { "ITEM0000000014", "18", "15", "10" }, + { "ITEM0000000015", "21", "16", "14" }, { "ITEM0000000016", "15", "15", "15" }, + { "ITEM0000000017", "19", "11", "12" }, { "ITEM0000000018", "14", "10", "11" }, + { "ITEM0000000019", "17", "11", "10" }, { "ITEM0000000020", "21", "8", "11" }, + { "ITEM0000000021", "19", "7", "11" }, { "ITEM0000000022", "19", "9", "11" }, + { "ITEM0000000023", "19", "13", "15" }, { "ITEM0000000024", "20", "15", "13" }, }; + private static final String[][] Q97_EXPECTED = { { "130", "82", "14" }, }; + private static final String[][] Q88_EXPECTED = { { "24", "24", "24" }, }; + private static final String[][] Q01_EXPECTED = { { "CUST000000000003" }, { "CUST000000000005" }, + { "CUST000000000006" }, { "CUST000000000009" }, { "CUST000000000014" }, { "CUST000000000014" }, + { "CUST000000000014" }, { "CUST000000000015" }, { "CUST000000000016" }, { "CUST000000000019" }, + { "CUST000000000020" }, { "CUST000000000021" }, { "CUST000000000023" }, { "CUST000000000023" }, + { "CUST000000000024" }, { "CUST000000000024" }, { "CUST000000000025" }, { "CUST000000000025" }, + { "CUST000000000026" }, { "CUST000000000027" }, { "CUST000000000029" }, }; + + /** For {@link TPCDSLikeExpectedRegenerator}. */ + public static Map queries() { + Map q = new LinkedHashMap<>(); + q.put("Q56", Q56); + q.put("Q60", Q60); + q.put("Q83", Q83); + q.put("Q97", Q97); + q.put("Q88", Q88); + q.put("Q01", Q01); + return q; + } + + @Test + public void testQ56() throws SQLException { + check("Q56", Q56, Q56_EXPECTED, "UNION ALL OVER 3 QUERIES"); + } + + @Test + public void testQ60() throws SQLException { + check("Q60", Q60, Q60_EXPECTED, "UNION ALL OVER 3 QUERIES"); + } + + @Test + public void testQ83() throws SQLException { + check("Q83", Q83, Q83_EXPECTED, "HASH BUILD"); + } + + @Test + public void testQ97() throws SQLException { + check("Q97", Q97, Q97_EXPECTED, new String[] { "SORT-MERGE-JOIN (FULL)" }, + new String[] { "SS_I", "CS_I" }); + } + + @Test + public void testQ88() throws SQLException { + check("Q88", Q88, Q88_EXPECTED, "HASH BUILD"); + } + + @Test + public void testQ01() throws SQLException { + check("Q01", Q01, Q01_EXPECTED, "HASH BUILD"); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeFixture.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeFixture.java new file mode 100644 index 00000000000..c50cbc555aa --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeFixture.java @@ -0,0 +1,922 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.tpcds; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Random; + +/** + * Shared deterministic TPC-DS derived fixture for {@link TPCDSLikeSingleChannelIT} and + * {@link TPCDSLikeCrossChannelIT}. + *

+ * The schema is a trimmed subset of the TPC-DS v4.0.0 schema, only the tables and columns touched + * by the adapted queries, with the following uniform Phoenix adaptation rules applied: + *

    + *
  • Surrogate keys ({@code *_sk}) become single-column {@code BIGINT NOT NULL PRIMARY KEY} + * columns on the dimension tables.
    • + *
  • Sales and returns fact tables use a composite primary key on + * {@code (*_item_sk, *_ticket_number)} or {@code *_order_number}. {@code inventory} uses + * {@code (inv_date_sk, inv_item_sk, inv_warehouse_sk)}.
    • + *
  • Fact tables are salted ({@code SALT_BUCKETS=4}).
    • + *
  • Foreign key columns are plain nullable columns. Referential consistency is guaranteed by this + * loader, not by the schema.
    • + *
+ *

+ * The data is generated with a fixed {@link Random} seed so that every adapted query has a stable + * reference answer. + *

+ * The fixture is materialized into two schemas that hold identical data: + *

    + *
  • {@link #SCHEMA} ({@value #SCHEMA}) -- base tables, no secondary indexes.
    • + *
  • {@link #SCHEMA_INDEXED} ({@value #SCHEMA_INDEXED}) -- the same, plus a covering global index + * per fact table.
    • + *
+ */ +public final class TPCDSLikeFixture { + + /** Base schema with no secondary indexes */ + public static final String SCHEMA = "TPCDS"; + /** Schema holding identical data plus a covering global index per fact table. */ + public static final String SCHEMA_INDEXED = "TPCDSI"; + + private static final long SEED = 42L; + + private static volatile boolean loaded = false; + + private TPCDSLikeFixture() { + } + + /** All base (dimension + fact) table short names, in dependency order, dimensions first. */ + static final String[] TABLES = + { "DATE_DIM", "TIME_DIM", "ITEM", "CUSTOMER_ADDRESS", "CUSTOMER_DEMOGRAPHICS", "INCOME_BAND", + "HOUSEHOLD_DEMOGRAPHICS", "CUSTOMER", "STORE", "CALL_CENTER", "CATALOG_PAGE", "WEB_SITE", + "WEB_PAGE", "WAREHOUSE", "SHIP_MODE", "PROMOTION", "REASON", "STORE_SALES", "STORE_RETURNS", + "CATALOG_SALES", "CATALOG_RETURNS", "WEB_SALES", "WEB_RETURNS", "INVENTORY" }; + + /** Create both schemas and populate them with deterministic data. */ + public static synchronized void load(Connection conn) throws SQLException { + if (loaded && tablesPopulated(conn)) { + return; + } + conn.setAutoCommit(false); + createTables(conn, SCHEMA); + createTables(conn, SCHEMA_INDEXED); + if (!tablesPopulated(conn)) { + loadData(conn, SCHEMA); + copyData(conn, SCHEMA, SCHEMA_INDEXED); + } + createIndexes(conn, SCHEMA_INDEXED); + conn.commit(); + loaded = true; + } + + /** + * Both schemas must carry data. A partial prior run or manual cleanup can leave the base schema + * populated while {@link #SCHEMA_INDEXED} is empty. + */ + private static boolean tablesPopulated(Connection conn) { + return rowsPresent(conn, SCHEMA) && rowsPresent(conn, SCHEMA_INDEXED); + } + + private static boolean rowsPresent(Connection conn, String s) { + try (Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("SELECT COUNT(*) FROM " + s + ".STORE_SALES")) { + return rs.next() && rs.getInt(1) > 0; + } catch (SQLException e) { + return false; + } + } + + private static void createTables(Connection conn, String s) throws SQLException { + try (Statement st = conn.createStatement()) { + for (String ddl : ddl(s)) { + st.execute(ddl); + } + } + } + + private static void createIndexes(Connection conn, String s) throws SQLException { + try (Statement st = conn.createStatement()) { + for (String ddl : indexDdl(s)) { + st.execute(ddl); + } + } + } + + private static String[] ddl(String s) { + return new String[] { + // Dimensions + "CREATE TABLE IF NOT EXISTS " + s + ".DATE_DIM (" + "d_date_sk BIGINT NOT NULL PRIMARY KEY," + + "d_date DATE, d_year INTEGER, d_moy INTEGER, d_dom INTEGER, d_qoy INTEGER," + + "d_dow INTEGER, d_week_seq INTEGER, d_day_name VARCHAR(16), d_quarter_name VARCHAR(8))", + "CREATE TABLE IF NOT EXISTS " + s + ".TIME_DIM (" + "t_time_sk BIGINT NOT NULL PRIMARY KEY," + + "t_hour INTEGER, t_minute INTEGER, t_am_pm VARCHAR(2), t_shift VARCHAR(16)," + + "t_meal_time VARCHAR(16))", + "CREATE TABLE IF NOT EXISTS " + s + ".ITEM (" + "i_item_sk BIGINT NOT NULL PRIMARY KEY," + + "i_item_id VARCHAR(16), i_item_desc VARCHAR(100), i_current_price DECIMAL(7,2)," + + "i_brand_id INTEGER, i_brand VARCHAR(50), i_class_id INTEGER, i_class VARCHAR(50)," + + "i_category_id INTEGER, i_category VARCHAR(50), i_manufact_id INTEGER," + + "i_manufact VARCHAR(50), i_manager_id INTEGER, i_size VARCHAR(20), i_color VARCHAR(20)," + + "i_units VARCHAR(10), i_product_name VARCHAR(50))", + "CREATE TABLE IF NOT EXISTS " + s + ".CUSTOMER_ADDRESS (" + + "ca_address_sk BIGINT NOT NULL PRIMARY KEY, ca_city VARCHAR(60), ca_county VARCHAR(30)," + + "ca_state VARCHAR(2), ca_zip VARCHAR(10), ca_country VARCHAR(20)," + + "ca_gmt_offset DECIMAL(5,2), ca_street_number VARCHAR(10), ca_street_name VARCHAR(60))", + "CREATE TABLE IF NOT EXISTS " + s + ".CUSTOMER_DEMOGRAPHICS (" + + "cd_demo_sk BIGINT NOT NULL PRIMARY KEY, cd_gender VARCHAR(1)," + + "cd_marital_status VARCHAR(1), cd_education_status VARCHAR(20)," + + "cd_purchase_estimate INTEGER, cd_credit_rating VARCHAR(10), cd_dep_count INTEGER," + + "cd_dep_employed_count INTEGER, cd_dep_college_count INTEGER)", + "CREATE TABLE IF NOT EXISTS " + s + ".INCOME_BAND (" + + "ib_income_band_sk BIGINT NOT NULL PRIMARY KEY, ib_lower_bound INTEGER," + + "ib_upper_bound INTEGER)", + "CREATE TABLE IF NOT EXISTS " + s + ".HOUSEHOLD_DEMOGRAPHICS (" + + "hd_demo_sk BIGINT NOT NULL PRIMARY KEY, hd_income_band_sk BIGINT," + + "hd_buy_potential VARCHAR(15), hd_dep_count INTEGER, hd_vehicle_count INTEGER)", + "CREATE TABLE IF NOT EXISTS " + s + ".CUSTOMER (" + + "c_customer_sk BIGINT NOT NULL PRIMARY KEY, c_customer_id VARCHAR(16)," + + "c_current_cdemo_sk BIGINT, c_current_hdemo_sk BIGINT, c_current_addr_sk BIGINT," + + "c_first_name VARCHAR(20), c_last_name VARCHAR(30), c_birth_country VARCHAR(20)," + + "c_birth_year INTEGER, c_preferred_cust_flag VARCHAR(1), c_salutation VARCHAR(10)," + + "c_email_address VARCHAR(50))", + "CREATE TABLE IF NOT EXISTS " + s + ".STORE (" + "s_store_sk BIGINT NOT NULL PRIMARY KEY," + + "s_store_id VARCHAR(16), s_store_name VARCHAR(50), s_company_id INTEGER," + + "s_company_name VARCHAR(50), s_state VARCHAR(2), s_zip VARCHAR(10)," + + "s_gmt_offset DECIMAL(5,2), s_city VARCHAR(60), s_county VARCHAR(30)," + + "s_manager VARCHAR(40), s_number_employees INTEGER)", + "CREATE TABLE IF NOT EXISTS " + s + ".CALL_CENTER (" + + "cc_call_center_sk BIGINT NOT NULL PRIMARY KEY, cc_call_center_id VARCHAR(16)," + + "cc_name VARCHAR(50), cc_manager VARCHAR(40), cc_mkt_id INTEGER, cc_class VARCHAR(50)," + + "cc_company INTEGER)", + "CREATE TABLE IF NOT EXISTS " + s + ".CATALOG_PAGE (" + + "cp_catalog_page_sk BIGINT NOT NULL PRIMARY KEY, cp_catalog_page_id VARCHAR(16)," + + "cp_catalog_number INTEGER, cp_catalog_page_number INTEGER, cp_department VARCHAR(50))", + "CREATE TABLE IF NOT EXISTS " + s + ".WEB_SITE (" + "web_site_sk BIGINT NOT NULL PRIMARY KEY," + + "web_site_id VARCHAR(16), web_name VARCHAR(50), web_company_id INTEGER," + + "web_company_name VARCHAR(50))", + "CREATE TABLE IF NOT EXISTS " + s + ".WEB_PAGE (" + + "wp_web_page_sk BIGINT NOT NULL PRIMARY KEY, wp_web_page_id VARCHAR(16)," + + "wp_char_count INTEGER, wp_type VARCHAR(50))", + "CREATE TABLE IF NOT EXISTS " + s + ".WAREHOUSE (" + + "w_warehouse_sk BIGINT NOT NULL PRIMARY KEY, w_warehouse_id VARCHAR(16)," + + "w_warehouse_name VARCHAR(50), w_state VARCHAR(2), w_country VARCHAR(20)," + + "w_gmt_offset DECIMAL(5,2))", + "CREATE TABLE IF NOT EXISTS " + s + ".SHIP_MODE (" + + "sm_ship_mode_sk BIGINT NOT NULL PRIMARY KEY, sm_ship_mode_id VARCHAR(16)," + + "sm_type VARCHAR(30), sm_carrier VARCHAR(20))", + "CREATE TABLE IF NOT EXISTS " + s + ".PROMOTION (" + + "p_promo_sk BIGINT NOT NULL PRIMARY KEY, p_promo_id VARCHAR(16)," + + "p_channel_email VARCHAR(1), p_channel_event VARCHAR(1), p_channel_tv VARCHAR(1)," + + "p_channel_dmail VARCHAR(1), p_channel_catalog VARCHAR(1))", + "CREATE TABLE IF NOT EXISTS " + s + ".REASON (" + "r_reason_sk BIGINT NOT NULL PRIMARY KEY," + + "r_reason_id VARCHAR(16), r_reason_desc VARCHAR(100))", + // Facts + "CREATE TABLE IF NOT EXISTS " + s + ".STORE_SALES (" + "ss_item_sk BIGINT NOT NULL," + + "ss_ticket_number BIGINT NOT NULL, ss_sold_date_sk BIGINT, ss_sold_time_sk BIGINT," + + "ss_customer_sk BIGINT, ss_cdemo_sk BIGINT, ss_hdemo_sk BIGINT, ss_addr_sk BIGINT," + + "ss_store_sk BIGINT, ss_promo_sk BIGINT, ss_quantity INTEGER, ss_sales_price DECIMAL(7,2)," + + "ss_ext_sales_price DECIMAL(7,2), ss_ext_discount_amt DECIMAL(7,2)," + + "ss_list_price DECIMAL(7,2), ss_ext_list_price DECIMAL(7,2), ss_coupon_amt DECIMAL(7,2)," + + "ss_net_profit DECIMAL(7,2), ss_net_paid DECIMAL(7,2) " + + "CONSTRAINT pk PRIMARY KEY (ss_item_sk, ss_ticket_number)) SALT_BUCKETS=4", + "CREATE TABLE IF NOT EXISTS " + s + ".STORE_RETURNS (" + "sr_item_sk BIGINT NOT NULL," + + "sr_ticket_number BIGINT NOT NULL, sr_returned_date_sk BIGINT, sr_customer_sk BIGINT," + + "sr_cdemo_sk BIGINT, sr_hdemo_sk BIGINT, sr_addr_sk BIGINT, sr_store_sk BIGINT," + + "sr_reason_sk BIGINT, sr_return_quantity INTEGER, sr_return_amt DECIMAL(7,2)," + + "sr_net_loss DECIMAL(7,2), sr_fee DECIMAL(7,2) " + + "CONSTRAINT pk PRIMARY KEY (sr_item_sk, sr_ticket_number)) SALT_BUCKETS=4", + "CREATE TABLE IF NOT EXISTS " + s + ".CATALOG_SALES (" + "cs_item_sk BIGINT NOT NULL," + + "cs_order_number BIGINT NOT NULL, cs_sold_date_sk BIGINT, cs_bill_customer_sk BIGINT," + + "cs_bill_cdemo_sk BIGINT, cs_ship_customer_sk BIGINT, cs_ship_addr_sk BIGINT," + + "cs_call_center_sk BIGINT, cs_catalog_page_sk BIGINT, cs_warehouse_sk BIGINT," + + "cs_ship_mode_sk BIGINT, cs_promo_sk BIGINT, cs_quantity INTEGER," + + "cs_sales_price DECIMAL(7,2), cs_ext_sales_price DECIMAL(7,2)," + + "cs_ext_discount_amt DECIMAL(7,2), cs_list_price DECIMAL(7,2)," + + "cs_ext_list_price DECIMAL(7,2), cs_coupon_amt DECIMAL(7,2), cs_net_profit DECIMAL(7,2)," + + "cs_net_paid DECIMAL(7,2) " + + "CONSTRAINT pk PRIMARY KEY (cs_item_sk, cs_order_number)) SALT_BUCKETS=4", + "CREATE TABLE IF NOT EXISTS " + s + ".CATALOG_RETURNS (" + "cr_item_sk BIGINT NOT NULL," + + "cr_order_number BIGINT NOT NULL, cr_returned_date_sk BIGINT," + + "cr_returning_customer_sk BIGINT, cr_call_center_sk BIGINT, cr_catalog_page_sk BIGINT," + + "cr_warehouse_sk BIGINT, cr_reason_sk BIGINT, cr_return_quantity INTEGER," + + "cr_return_amount DECIMAL(7,2), cr_net_loss DECIMAL(7,2) " + + "CONSTRAINT pk PRIMARY KEY (cr_item_sk, cr_order_number)) SALT_BUCKETS=4", + "CREATE TABLE IF NOT EXISTS " + s + ".WEB_SALES (" + "ws_item_sk BIGINT NOT NULL," + + "ws_order_number BIGINT NOT NULL, ws_sold_date_sk BIGINT, ws_sold_time_sk BIGINT," + + "ws_bill_customer_sk BIGINT, ws_ship_customer_sk BIGINT, ws_ship_addr_sk BIGINT," + + "ws_web_site_sk BIGINT, ws_web_page_sk BIGINT, ws_warehouse_sk BIGINT," + + "ws_ship_mode_sk BIGINT, ws_promo_sk BIGINT, ws_quantity INTEGER," + + "ws_sales_price DECIMAL(7,2), ws_ext_sales_price DECIMAL(7,2)," + + "ws_ext_discount_amt DECIMAL(7,2), ws_list_price DECIMAL(7,2)," + + "ws_ext_list_price DECIMAL(7,2), ws_coupon_amt DECIMAL(7,2), ws_net_profit DECIMAL(7,2)," + + "ws_net_paid DECIMAL(7,2) " + + "CONSTRAINT pk PRIMARY KEY (ws_item_sk, ws_order_number)) SALT_BUCKETS=4", + "CREATE TABLE IF NOT EXISTS " + s + ".WEB_RETURNS (" + "wr_item_sk BIGINT NOT NULL," + + "wr_order_number BIGINT NOT NULL, wr_returned_date_sk BIGINT," + + "wr_returning_customer_sk BIGINT, wr_web_page_sk BIGINT, wr_reason_sk BIGINT," + + "wr_return_quantity INTEGER, wr_return_amt DECIMAL(7,2), wr_net_loss DECIMAL(7,2) " + + "CONSTRAINT pk PRIMARY KEY (wr_item_sk, wr_order_number)) SALT_BUCKETS=4", + "CREATE TABLE IF NOT EXISTS " + s + ".INVENTORY (" + "inv_date_sk BIGINT NOT NULL," + + "inv_item_sk BIGINT NOT NULL, inv_warehouse_sk BIGINT NOT NULL," + + "inv_quantity_on_hand INTEGER " + + "CONSTRAINT pk PRIMARY KEY (inv_date_sk, inv_item_sk, inv_warehouse_sk)) SALT_BUCKETS=4" }; + } + + private static String[] indexDdl(String s) { + String suffix = "_I"; + return new String[] { + "CREATE INDEX IF NOT EXISTS SS" + suffix + " ON " + s + + ".STORE_SALES (ss_sold_date_sk, ss_item_sk) INCLUDE (ss_store_sk, ss_customer_sk," + + " ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_discount_amt, ss_list_price," + + " ss_net_profit, ss_net_paid)", + "CREATE INDEX IF NOT EXISTS CS" + suffix + " ON " + s + + ".CATALOG_SALES (cs_sold_date_sk, cs_item_sk) INCLUDE (cs_bill_customer_sk," + + " cs_warehouse_sk, cs_quantity, cs_sales_price, cs_ext_sales_price, cs_ext_discount_amt," + + " cs_list_price, cs_net_profit, cs_net_paid)", + "CREATE INDEX IF NOT EXISTS WS" + suffix + " ON " + s + + ".WEB_SALES (ws_sold_date_sk, ws_item_sk) INCLUDE (ws_bill_customer_sk, ws_warehouse_sk," + + " ws_quantity, ws_sales_price, ws_ext_sales_price, ws_ext_discount_amt, ws_list_price," + + " ws_net_profit, ws_net_paid)", + "CREATE INDEX IF NOT EXISTS INV" + suffix + " ON " + s + + ".INVENTORY (inv_item_sk, inv_date_sk) INCLUDE (inv_warehouse_sk, inv_quantity_on_hand)" }; + } + + static final int[] YEARS = { 2000, 2001 }; + static final String[] CATEGORIES = + { "Books", "Home", "Electronics", "Sports", "Jewelry", "Music" }; + static final String[] STATES = { "CA", "TX", "NY", "WA", "OR" }; + static final String[] GENDERS = { "M", "F" }; + static final String[] MARITAL = { "M", "S", "D", "W", "U" }; + static final String[] EDUCATION = + { "Primary", "Secondary", "College", "2 yr Degree", "4 yr Degree", "Advanced Degree" }; + static final String[] BUY_POTENTIAL = + { ">10000", "5001-10000", "1001-5000", "501-1000", "0-500", "Unknown" }; + static final String[] DAY_NAMES = + { "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; + static final double[] GMT_OFFSETS = { -5, -6, -7, -8 }; + + private static final int N_ITEMS = 24; + private static final int N_CUSTOMERS = 30; + private static final int N_ADDR = 20; + private static final int N_CDEMO = 12; + private static final int N_HDEMO = 12; + private static final int N_STORES = 6; + private static final int N_CALL_CENTERS = 4; + private static final int N_CATALOG_PAGES = 6; + private static final int N_WEB_SITES = 4; + private static final int N_WEB_PAGES = 6; + private static final int N_WAREHOUSES = 4; + private static final int N_SHIP_MODES = 5; + private static final int N_PROMOS = 10; + private static final int N_REASONS = 6; + + // date_dim: 24 rows = 12 months x 2 years, d_date_sk = 1..24. + private static final int N_DATES = YEARS.length * 12; + private static final int N_TIMES = 12; + + private static void loadData(Connection conn, String s) throws SQLException { + Random rnd = new Random(SEED); + loadDateDim(conn, s); + loadTimeDim(conn, s); + loadItem(conn, s, rnd); + loadCustomerAddress(conn, s, rnd); + loadCustomerDemographics(conn, s); + loadIncomeBand(conn, s); + loadHouseholdDemographics(conn, s, rnd); + loadCustomer(conn, s, rnd); + loadStore(conn, s, rnd); + loadCallCenter(conn, s); + loadCatalogPage(conn, s); + loadWebSite(conn, s); + loadWebPage(conn, s); + loadWarehouse(conn, s, rnd); + loadShipMode(conn, s); + loadPromotion(conn, s, rnd); + loadReason(conn, s); + conn.commit(); + loadStoreSales(conn, s, rnd); + loadStoreReturns(conn, s, rnd); + loadCatalogSales(conn, s, rnd); + loadCatalogReturns(conn, s, rnd); + loadWebSales(conn, s, rnd); + loadWebReturns(conn, s, rnd); + loadInventory(conn, s, rnd); + conn.commit(); + } + + private static void copyData(Connection conn, String from, String to) throws SQLException { + try (Statement st = conn.createStatement()) { + for (String t : TABLES) { + st.executeUpdate("UPSERT INTO " + to + "." + t + " SELECT * FROM " + from + "." + t); + } + } + conn.commit(); + } + + private static void loadDateDim(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".DATE_DIM (d_date_sk, d_date, d_year, d_moy, d_dom, d_qoy," + + " d_dow, d_week_seq, d_day_name, d_quarter_name) VALUES (?,?,?,?,?,?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int sk = 0; + for (int yi = 0; yi < YEARS.length; yi++) { + int year = YEARS[yi]; + for (int moy = 1; moy <= 12; moy++) { + sk++; + int dom = 15; + int qoy = (moy - 1) / 3 + 1; + int dow = (sk % 7); + int weekSeq = (year - 2000) * 52 + moy * 4; + ps.setLong(1, sk); + ps.setDate(2, java.sql.Date.valueOf(String.format("%04d-%02d-%02d", year, moy, dom))); + ps.setInt(3, year); + ps.setInt(4, moy); + ps.setInt(5, dom); + ps.setInt(6, qoy); + ps.setInt(7, dow); + ps.setInt(8, weekSeq); + ps.setString(9, DAY_NAMES[dow]); + ps.setString(10, year + "Q" + qoy); + ps.executeUpdate(); + } + } + } + } + + private static void loadTimeDim(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".TIME_DIM (t_time_sk, t_hour, t_minute, t_am_pm, t_shift," + + " t_meal_time) VALUES (?,?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_TIMES; i++) { + int hour = (i * 2) % 24; + ps.setLong(1, i); + ps.setInt(2, hour); + ps.setInt(3, 0); + ps.setString(4, hour < 12 ? "AM" : "PM"); + ps.setString(5, hour < 8 ? "third" : hour < 16 ? "first" : "second"); + ps.setString(6, + hour >= 6 && hour < 10 ? "breakfast" + : hour >= 11 && hour < 14 ? "lunch" + : hour >= 17 && hour < 21 ? "dinner" + : null); + ps.executeUpdate(); + } + } + } + + private static void loadItem(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".ITEM (i_item_sk, i_item_id, i_item_desc, i_current_price," + + " i_brand_id, i_brand, i_class_id, i_class, i_category_id, i_category, i_manufact_id," + + " i_manufact, i_manager_id, i_size, i_color, i_units, i_product_name)" + + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + String[] sizes = { "small", "medium", "large", "petite", "economy", "N/A" }; + String[] colors = { "red", "blue", "green", "white", "black", "almond" }; + String[] units = { "Each", "Dozen", "Case", "Box", "Pallet" }; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_ITEMS; i++) { + int catId = (i - 1) % CATEGORIES.length; + int classId = (i - 1) % 5 + 1; + int brandId = 1000 + ((i - 1) % 8); + int manufactId = (i - 1) % 5 + 1; + int managerId = (i - 1) % 10 + 1; + ps.setLong(1, i); + ps.setString(2, String.format("ITEM%010d", i)); + ps.setString(3, "Item description " + i); + ps.setBigDecimal(4, bd(10 + (i % 90) + 0.99)); + ps.setInt(5, brandId); + ps.setString(6, "brand#" + brandId); + ps.setInt(7, classId); + ps.setString(8, "class#" + classId); + ps.setInt(9, catId + 1); + ps.setString(10, CATEGORIES[catId]); + ps.setInt(11, manufactId); + ps.setString(12, "manufact#" + manufactId); + ps.setInt(13, managerId); + ps.setString(14, sizes[i % sizes.length]); + ps.setString(15, colors[i % colors.length]); + ps.setString(16, units[i % units.length]); + ps.setString(17, "product#" + i); + ps.executeUpdate(); + } + } + } + + private static void loadCustomerAddress(Connection conn, String s, Random rnd) + throws SQLException { + String sql = "UPSERT INTO " + s + ".CUSTOMER_ADDRESS (ca_address_sk, ca_city, ca_county," + + " ca_state, ca_zip, ca_country, ca_gmt_offset, ca_street_number, ca_street_name)" + + " VALUES (?,?,?,?,?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_ADDR; i++) { + String state = STATES[(i - 1) % STATES.length]; + ps.setLong(1, i); + ps.setString(2, "City" + ((i - 1) % 7)); + ps.setString(3, "County" + ((i - 1) % 4)); + ps.setString(4, state); + ps.setString(5, String.format("%05d", 10000 + i)); + ps.setString(6, "United States"); + ps.setBigDecimal(7, bd(GMT_OFFSETS[(i - 1) % GMT_OFFSETS.length])); + ps.setString(8, Integer.toString(100 + i)); + ps.setString(9, "Street " + i); + ps.executeUpdate(); + } + } + } + + private static void loadCustomerDemographics(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".CUSTOMER_DEMOGRAPHICS (cd_demo_sk, cd_gender," + + " cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating," + + " cd_dep_count, cd_dep_employed_count, cd_dep_college_count) VALUES (?,?,?,?,?,?,?,?,?)"; + String[] credit = { "Low Risk", "Good", "High Risk", "Unknown" }; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_CDEMO; i++) { + ps.setLong(1, i); + ps.setString(2, GENDERS[(i - 1) % GENDERS.length]); + ps.setString(3, MARITAL[(i - 1) % MARITAL.length]); + ps.setString(4, EDUCATION[(i - 1) % EDUCATION.length]); + ps.setInt(5, 500 * (((i - 1) % 6) + 1)); + ps.setString(6, credit[(i - 1) % credit.length]); + ps.setInt(7, (i - 1) % 4); + ps.setInt(8, (i - 1) % 3); + ps.setInt(9, (i - 1) % 2); + ps.executeUpdate(); + } + } + } + + private static void loadIncomeBand(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".INCOME_BAND (ib_income_band_sk, ib_lower_bound," + + " ib_upper_bound) VALUES (?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= 10; i++) { + ps.setLong(1, i); + ps.setInt(2, (i - 1) * 10000); + ps.setInt(3, i * 10000); + ps.executeUpdate(); + } + } + } + + private static void loadHouseholdDemographics(Connection conn, String s, Random rnd) + throws SQLException { + String sql = "UPSERT INTO " + s + ".HOUSEHOLD_DEMOGRAPHICS (hd_demo_sk, hd_income_band_sk," + + " hd_buy_potential, hd_dep_count, hd_vehicle_count) VALUES (?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_HDEMO; i++) { + ps.setLong(1, i); + ps.setLong(2, ((i - 1) % 10) + 1); + ps.setString(3, BUY_POTENTIAL[(i - 1) % BUY_POTENTIAL.length]); + ps.setInt(4, (i - 1) % 5); + ps.setInt(5, (i - 1) % 4); + ps.executeUpdate(); + } + } + } + + private static void loadCustomer(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".CUSTOMER (c_customer_sk, c_customer_id, c_current_cdemo_sk," + + " c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country," + + " c_birth_year, c_preferred_cust_flag, c_salutation, c_email_address)" + + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"; + String[] countries = { "UNITED STATES", "CANADA", "MEXICO", "BRAZIL" }; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_CUSTOMERS; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("CUST%012d", i)); + ps.setLong(3, ((i - 1) % N_CDEMO) + 1); + ps.setLong(4, ((i - 1) % N_HDEMO) + 1); + ps.setLong(5, ((i - 1) % N_ADDR) + 1); + ps.setString(6, "First" + i); + ps.setString(7, "Last" + i); + ps.setString(8, countries[(i - 1) % countries.length]); + ps.setInt(9, 1950 + (i % 40)); + ps.setString(10, (i % 2 == 0) ? "Y" : "N"); + ps.setString(11, (i % 2 == 0) ? "Mr." : "Ms."); + ps.setString(12, "cust" + i + "@example.com"); + ps.executeUpdate(); + } + } + } + + private static void loadStore(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".STORE (s_store_sk, s_store_id, s_store_name, s_company_id," + + " s_company_name, s_state, s_zip, s_gmt_offset, s_city, s_county, s_manager," + + " s_number_employees) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_STORES; i++) { + String state = STATES[(i - 1) % STATES.length]; + ps.setLong(1, i); + ps.setString(2, String.format("STORE%010d", i)); + ps.setString(3, "Store " + i); + ps.setInt(4, 1); + ps.setString(5, "company#1"); + ps.setString(6, state); + ps.setString(7, String.format("%05d", 20000 + i)); + ps.setBigDecimal(8, bd(GMT_OFFSETS[(i - 1) % GMT_OFFSETS.length])); + ps.setString(9, "City" + ((i - 1) % 7)); + ps.setString(10, "County" + ((i - 1) % 4)); + ps.setString(11, "Manager" + i); + ps.setInt(12, 100 + i * 10); + ps.executeUpdate(); + } + } + } + + private static void loadCallCenter(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".CALL_CENTER (cc_call_center_sk, cc_call_center_id, cc_name," + + " cc_manager, cc_mkt_id, cc_class, cc_company) VALUES (?,?,?,?,?,?,?)"; + String[] classes = { "small", "medium", "large" }; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_CALL_CENTERS; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("CC%014d", i)); + ps.setString(3, "Call Center " + i); + ps.setString(4, "CCManager" + i); + ps.setInt(5, (i - 1) % 3 + 1); + ps.setString(6, classes[(i - 1) % classes.length]); + ps.setInt(7, 1); + ps.executeUpdate(); + } + } + } + + private static void loadCatalogPage(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".CATALOG_PAGE (cp_catalog_page_sk, cp_catalog_page_id," + + " cp_catalog_number, cp_catalog_page_number, cp_department) VALUES (?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_CATALOG_PAGES; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("CP%014d", i)); + ps.setInt(3, (i - 1) % 3 + 1); + ps.setInt(4, i); + ps.setString(5, "DEPARTMENT"); + ps.executeUpdate(); + } + } + } + + private static void loadWebSite(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".WEB_SITE (web_site_sk, web_site_id, web_name," + + " web_company_id, web_company_name) VALUES (?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_WEB_SITES; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("WS%014d", i)); + ps.setString(3, "Web Site " + i); + ps.setInt(4, 1); + ps.setString(5, "company#1"); + ps.executeUpdate(); + } + } + } + + private static void loadWebPage(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".WEB_PAGE (wp_web_page_sk, wp_web_page_id, wp_char_count," + + " wp_type) VALUES (?,?,?,?)"; + String[] types = { "welcome", "protected", "feedback", "general", "ad", "order" }; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_WEB_PAGES; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("WP%014d", i)); + ps.setInt(3, 1000 + i * 100); + ps.setString(4, types[(i - 1) % types.length]); + ps.executeUpdate(); + } + } + } + + private static void loadWarehouse(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".WAREHOUSE (w_warehouse_sk, w_warehouse_id," + + " w_warehouse_name, w_state, w_country, w_gmt_offset) VALUES (?,?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_WAREHOUSES; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("WH%014d", i)); + ps.setString(3, "Warehouse " + i); + ps.setString(4, STATES[(i - 1) % STATES.length]); + ps.setString(5, "United States"); + ps.setBigDecimal(6, bd(GMT_OFFSETS[(i - 1) % GMT_OFFSETS.length])); + ps.executeUpdate(); + } + } + } + + private static void loadShipMode(Connection conn, String s) throws SQLException { + String sql = "UPSERT INTO " + s + ".SHIP_MODE (sm_ship_mode_sk, sm_ship_mode_id, sm_type," + + " sm_carrier) VALUES (?,?,?,?)"; + String[] types = { "EXPRESS", "NEXT DAY", "OVERNIGHT", "TWO DAY", "LIBRARY" }; + String[] carriers = { "DHL", "FEDEX", "UPS", "USPS", "ZHOU" }; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_SHIP_MODES; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("SM%014d", i)); + ps.setString(3, types[(i - 1) % types.length]); + ps.setString(4, carriers[(i - 1) % carriers.length]); + ps.executeUpdate(); + } + } + } + + private static void loadPromotion(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".PROMOTION (p_promo_sk, p_promo_id, p_channel_email," + + " p_channel_event, p_channel_tv, p_channel_dmail, p_channel_catalog) VALUES (?,?,?,?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_PROMOS; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("PROMO%010d", i)); + ps.setString(3, (i % 2 == 0) ? "Y" : "N"); + ps.setString(4, (i % 3 == 0) ? "Y" : "N"); + ps.setString(5, (i % 2 == 1) ? "Y" : "N"); + ps.setString(6, (i % 4 == 0) ? "Y" : "N"); + ps.setString(7, (i % 5 == 0) ? "Y" : "N"); + ps.executeUpdate(); + } + } + } + + private static void loadReason(Connection conn, String s) throws SQLException { + String sql = + "UPSERT INTO " + s + ".REASON (r_reason_sk, r_reason_id, r_reason_desc)" + " VALUES (?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int i = 1; i <= N_REASONS; i++) { + ps.setLong(1, i); + ps.setString(2, String.format("REASON%010d", i)); + ps.setString(3, "Reason " + i); + ps.executeUpdate(); + } + } + } + + private static void loadStoreSales(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".STORE_SALES (ss_item_sk, ss_ticket_number, ss_sold_date_sk," + + " ss_sold_time_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk," + + " ss_promo_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_discount_amt," + + " ss_list_price, ss_ext_list_price, ss_coupon_amt, ss_net_profit, ss_net_paid)" + + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + long ticket = 0; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int item = 1; item <= N_ITEMS; item++) { + // Each item sold on a handful of dates, giving broad coverage of (year, moy). + for (int dateSk = 1; dateSk <= N_DATES; dateSk += 2) { + ticket++; + int qty = 1 + rnd.nextInt(20); + double salesPrice = 5 + rnd.nextInt(95) + 0.5; + double listPrice = salesPrice + rnd.nextInt(20); + long cust = 1 + ((item + dateSk) % N_CUSTOMERS); + ps.setLong(1, item); + ps.setLong(2, ticket); + ps.setLong(3, dateSk); + ps.setLong(4, 1 + (ticket % N_TIMES)); + ps.setLong(5, cust); + ps.setLong(6, 1 + ((cust - 1) % N_CDEMO)); + ps.setLong(7, 1 + ((cust - 1) % N_HDEMO)); + ps.setLong(8, 1 + ((cust - 1) % N_ADDR)); + ps.setLong(9, 1 + (int) (ticket % N_STORES)); + ps.setLong(10, 1 + (int) (ticket % N_PROMOS)); + ps.setInt(11, qty); + ps.setBigDecimal(12, bd(salesPrice)); + ps.setBigDecimal(13, bd(salesPrice * qty)); + ps.setBigDecimal(14, bd(rnd.nextInt(50))); + ps.setBigDecimal(15, bd(listPrice)); + ps.setBigDecimal(16, bd(listPrice * qty)); + ps.setBigDecimal(17, bd(rnd.nextInt(20))); + ps.setBigDecimal(18, bd((salesPrice - 3) * qty)); + ps.setBigDecimal(19, bd(salesPrice * qty)); + ps.executeUpdate(); + } + } + } + conn.commit(); + } + + private static void loadStoreReturns(Connection conn, String s, Random rnd) throws SQLException { + // Return every other store_sales row (even tickets), reusing its (item, ticket) so the natural + // join key lines up with store_sales. + String sql = "UPSERT INTO " + s + ".STORE_RETURNS (sr_item_sk, sr_ticket_number," + + " sr_returned_date_sk, sr_customer_sk, sr_cdemo_sk, sr_hdemo_sk, sr_addr_sk, sr_store_sk," + + " sr_reason_sk, sr_return_quantity, sr_return_amt, sr_net_loss, sr_fee)" + + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"; + long ticket = 0; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int item = 1; item <= N_ITEMS; item++) { + for (int dateSk = 1; dateSk <= N_DATES; dateSk += 2) { + ticket++; + if (ticket % 2 != 0) { + continue; + } + long cust = 1 + ((item + dateSk) % N_CUSTOMERS); + int retDate = Math.min(N_DATES, dateSk + 1); + ps.setLong(1, item); + ps.setLong(2, ticket); + ps.setLong(3, retDate); + ps.setLong(4, cust); + ps.setLong(5, 1 + ((cust - 1) % N_CDEMO)); + ps.setLong(6, 1 + ((cust - 1) % N_HDEMO)); + ps.setLong(7, 1 + ((cust - 1) % N_ADDR)); + ps.setLong(8, 1 + (int) (ticket % N_STORES)); + ps.setLong(9, 1 + (int) (ticket % N_REASONS)); + ps.setInt(10, 1 + rnd.nextInt(5)); + ps.setBigDecimal(11, bd(5 + rnd.nextInt(50))); + ps.setBigDecimal(12, bd(rnd.nextInt(30))); + ps.setBigDecimal(13, bd(rnd.nextInt(10))); + ps.executeUpdate(); + } + } + } + conn.commit(); + } + + private static void loadCatalogSales(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".CATALOG_SALES (cs_item_sk, cs_order_number," + + " cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_ship_customer_sk," + + " cs_ship_addr_sk, cs_call_center_sk, cs_catalog_page_sk, cs_warehouse_sk, cs_ship_mode_sk," + + " cs_promo_sk, cs_quantity, cs_sales_price, cs_ext_sales_price, cs_ext_discount_amt," + + " cs_list_price, cs_ext_list_price, cs_coupon_amt, cs_net_profit, cs_net_paid)" + + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + long order = 0; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int item = 1; item <= N_ITEMS; item++) { + for (int dateSk = 2; dateSk <= N_DATES; dateSk += 3) { + order++; + int qty = 1 + rnd.nextInt(20); + double salesPrice = 5 + rnd.nextInt(95) + 0.5; + double listPrice = salesPrice + rnd.nextInt(20); + long cust = 1 + ((item * 2 + dateSk) % N_CUSTOMERS); + ps.setLong(1, item); + ps.setLong(2, order); + ps.setLong(3, dateSk); + ps.setLong(4, cust); + ps.setLong(5, 1 + ((cust - 1) % N_CDEMO)); + ps.setLong(6, 1 + (cust % N_CUSTOMERS)); + ps.setLong(7, 1 + ((cust - 1) % N_ADDR)); + ps.setLong(8, 1 + (int) (order % N_CALL_CENTERS)); + ps.setLong(9, 1 + (int) (order % N_CATALOG_PAGES)); + ps.setLong(10, 1 + (int) (order % N_WAREHOUSES)); + ps.setLong(11, 1 + (int) (order % N_SHIP_MODES)); + ps.setLong(12, 1 + (int) (order % N_PROMOS)); + ps.setInt(13, qty); + ps.setBigDecimal(14, bd(salesPrice)); + ps.setBigDecimal(15, bd(salesPrice * qty)); + ps.setBigDecimal(16, bd(rnd.nextInt(50))); + ps.setBigDecimal(17, bd(listPrice)); + ps.setBigDecimal(18, bd(listPrice * qty)); + ps.setBigDecimal(19, bd(rnd.nextInt(20))); + ps.setBigDecimal(20, bd((salesPrice - 3) * qty)); + ps.setBigDecimal(21, bd(salesPrice * qty)); + ps.executeUpdate(); + } + } + } + conn.commit(); + } + + private static void loadCatalogReturns(Connection conn, String s, Random rnd) + throws SQLException { + String sql = "UPSERT INTO " + s + ".CATALOG_RETURNS (cr_item_sk, cr_order_number," + + " cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_catalog_page_sk," + + " cr_warehouse_sk, cr_reason_sk, cr_return_quantity, cr_return_amount, cr_net_loss)" + + " VALUES (?,?,?,?,?,?,?,?,?,?,?)"; + long order = 0; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int item = 1; item <= N_ITEMS; item++) { + for (int dateSk = 2; dateSk <= N_DATES; dateSk += 3) { + order++; + if (order % 2 != 0) { + continue; + } + long cust = 1 + ((item * 2 + dateSk) % N_CUSTOMERS); + int retDate = Math.min(N_DATES, dateSk + 1); + ps.setLong(1, item); + ps.setLong(2, order); + ps.setLong(3, retDate); + ps.setLong(4, cust); + ps.setLong(5, 1 + (int) (order % N_CALL_CENTERS)); + ps.setLong(6, 1 + (int) (order % N_CATALOG_PAGES)); + ps.setLong(7, 1 + (int) (order % N_WAREHOUSES)); + ps.setLong(8, 1 + (int) (order % N_REASONS)); + ps.setInt(9, 1 + rnd.nextInt(5)); + ps.setBigDecimal(10, bd(5 + rnd.nextInt(50))); + ps.setBigDecimal(11, bd(rnd.nextInt(10))); + ps.executeUpdate(); + } + } + } + conn.commit(); + } + + private static void loadWebSales(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".WEB_SALES (ws_item_sk, ws_order_number, ws_sold_date_sk," + + " ws_sold_time_sk, ws_bill_customer_sk, ws_ship_customer_sk, ws_ship_addr_sk," + + " ws_web_site_sk, ws_web_page_sk, ws_warehouse_sk, ws_ship_mode_sk, ws_promo_sk," + + " ws_quantity, ws_sales_price, ws_ext_sales_price, ws_ext_discount_amt, ws_list_price," + + " ws_ext_list_price, ws_coupon_amt, ws_net_profit, ws_net_paid)" + + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + long order = 0; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int item = 1; item <= N_ITEMS; item++) { + for (int dateSk = 3; dateSk <= N_DATES; dateSk += 3) { + order++; + int qty = 1 + rnd.nextInt(20); + double salesPrice = 5 + rnd.nextInt(95) + 0.5; + double listPrice = salesPrice + rnd.nextInt(20); + long cust = 1 + ((item * 3 + dateSk) % N_CUSTOMERS); + ps.setLong(1, item); + ps.setLong(2, order); + ps.setLong(3, dateSk); + ps.setLong(4, 1 + (order % N_TIMES)); + ps.setLong(5, cust); + ps.setLong(6, 1 + (cust % N_CUSTOMERS)); + ps.setLong(7, 1 + ((cust - 1) % N_ADDR)); + ps.setLong(8, 1 + (int) (order % N_WEB_SITES)); + ps.setLong(9, 1 + (int) (order % N_WEB_PAGES)); + ps.setLong(10, 1 + (int) (order % N_WAREHOUSES)); + ps.setLong(11, 1 + (int) (order % N_SHIP_MODES)); + ps.setLong(12, 1 + (int) (order % N_PROMOS)); + ps.setInt(13, qty); + ps.setBigDecimal(14, bd(salesPrice)); + ps.setBigDecimal(15, bd(salesPrice * qty)); + ps.setBigDecimal(16, bd(rnd.nextInt(50))); + ps.setBigDecimal(17, bd(listPrice)); + ps.setBigDecimal(18, bd(listPrice * qty)); + ps.setBigDecimal(19, bd(rnd.nextInt(20))); + ps.setBigDecimal(20, bd((salesPrice - 3) * qty)); + ps.setBigDecimal(21, bd(salesPrice * qty)); + ps.executeUpdate(); + } + } + } + conn.commit(); + } + + private static void loadWebReturns(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".WEB_RETURNS (wr_item_sk, wr_order_number," + + " wr_returned_date_sk, wr_returning_customer_sk, wr_web_page_sk, wr_reason_sk," + + " wr_return_quantity, wr_return_amt, wr_net_loss) VALUES (?,?,?,?,?,?,?,?,?)"; + long order = 0; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + for (int item = 1; item <= N_ITEMS; item++) { + for (int dateSk = 3; dateSk <= N_DATES; dateSk += 3) { + order++; + if (order % 2 != 0) { + continue; + } + long cust = 1 + ((item * 3 + dateSk) % N_CUSTOMERS); + int retDate = Math.min(N_DATES, dateSk + 1); + ps.setLong(1, item); + ps.setLong(2, order); + ps.setLong(3, retDate); + ps.setLong(4, cust); + ps.setLong(5, 1 + (int) (order % N_WEB_PAGES)); + ps.setLong(6, 1 + (int) (order % N_REASONS)); + ps.setInt(7, 1 + rnd.nextInt(5)); + ps.setBigDecimal(8, bd(5 + rnd.nextInt(50))); + ps.setBigDecimal(9, bd(rnd.nextInt(10))); + ps.executeUpdate(); + } + } + } + conn.commit(); + } + + private static void loadInventory(Connection conn, String s, Random rnd) throws SQLException { + String sql = "UPSERT INTO " + s + ".INVENTORY (inv_date_sk, inv_item_sk, inv_warehouse_sk," + + " inv_quantity_on_hand) VALUES (?,?,?,?)"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + // Weekly-ish snapshots: every 3rd date, all items, all warehouses. + for (int dateSk = 1; dateSk <= N_DATES; dateSk += 3) { + for (int item = 1; item <= N_ITEMS; item++) { + for (int wh = 1; wh <= N_WAREHOUSES; wh++) { + ps.setLong(1, dateSk); + ps.setLong(2, item); + ps.setLong(3, wh); + ps.setInt(4, 50 + rnd.nextInt(600)); + ps.executeUpdate(); + } + } + } + } + conn.commit(); + } + + private static java.math.BigDecimal bd(double v) { + return new java.math.BigDecimal(v).setScale(2, java.math.RoundingMode.HALF_UP); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeSingleChannelIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeSingleChannelIT.java new file mode 100644 index 00000000000..5c91e48bd7e --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeSingleChannelIT.java @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.tpcds; + +import java.sql.SQLException; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.phoenix.end2end.ParallelStatsDisabledTest; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * TPC-DS derived integration tests over a single sales and returns channel. Each {@code @Test} + * adapts a public TPC-DS query to Phoenix, asserts the full ordered result, and asserts the + * rendered EXPLAIN plan. + */ +@Category(ParallelStatsDisabledTest.class) +@RunWith(Parameterized.class) +public class TPCDSLikeSingleChannelIT extends TPCDSLikeBaseIT { + + public TPCDSLikeSingleChannelIT(String label, String schema, boolean noIndex) { + super(label, schema, noIndex); + } + + @Parameters(name = "{0}") + public static Collection params() { + return indexParameters(); + } + + @BeforeClass + public static synchronized void setup() throws SQLException { + loadFixture(); + } + + // TPC-DS Q3: store sales discount by brand for a manufacturer/month. + static final String Q03 = "SELECT d.d_year, i.i_brand_id brand_id, i.i_brand brand," + + " SUM(ss.ss_ext_discount_amt) sum_agg" + + " FROM TPCDS.date_dim d, TPCDS.store_sales ss, TPCDS.item i" + + " WHERE d.d_date_sk = ss.ss_sold_date_sk AND ss.ss_item_sk = i.i_item_sk" + + " AND i.i_manufact_id = 1 AND d.d_moy = 11" + " GROUP BY d.d_year, i.i_brand, i.i_brand_id" + + " ORDER BY d.d_year, sum_agg DESC, brand_id LIMIT 100"; + + // TPC-DS Q7: average sale measures by item for a gender/marital/promo segment. + static final String Q07 = "SELECT i.i_item_id, AVG(ss.ss_quantity) agg1," + + " AVG(ss.ss_list_price) agg2, AVG(ss.ss_coupon_amt) agg3, AVG(ss.ss_sales_price) agg4" + + " FROM TPCDS.store_sales ss, TPCDS.customer_demographics cd, TPCDS.date_dim d," + + " TPCDS.item i, TPCDS.promotion p" + + " WHERE ss.ss_sold_date_sk = d.d_date_sk AND ss.ss_item_sk = i.i_item_sk" + + " AND ss.ss_cdemo_sk = cd.cd_demo_sk AND ss.ss_promo_sk = p.p_promo_sk" + + " AND cd.cd_gender = 'M' AND cd.cd_marital_status = 'S'" + + " AND (p.p_channel_email = 'N' OR p.p_channel_event = 'N') AND d.d_year = 2000" + + " GROUP BY i.i_item_id ORDER BY i.i_item_id LIMIT 100"; + + // TPC-DS Q42: store ext sales price by category for a manager/month/year. + static final String Q42 = + "SELECT d.d_year, i.i_category_id, i.i_category," + " SUM(ss.ss_ext_sales_price) sum_price" + + " FROM TPCDS.date_dim d, TPCDS.store_sales ss, TPCDS.item i" + + " WHERE d.d_date_sk = ss.ss_sold_date_sk AND ss.ss_item_sk = i.i_item_sk" + + " AND i.i_manager_id = 1 AND d.d_moy = 11 AND d.d_year = 2000" + + " GROUP BY d.d_year, i.i_category_id, i.i_category" + + " ORDER BY sum_price DESC, d.d_year, i.i_category_id, i.i_category LIMIT 100"; + + // TPC-DS Q52: store ext sales price by brand for a manager/month/year. + static final String Q52 = "SELECT d.d_year, i.i_brand_id brand_id, i.i_brand brand," + + " SUM(ss.ss_ext_sales_price) ext_price" + + " FROM TPCDS.date_dim d, TPCDS.store_sales ss, TPCDS.item i" + + " WHERE d.d_date_sk = ss.ss_sold_date_sk AND ss.ss_item_sk = i.i_item_sk" + + " AND i.i_manager_id = 1 AND d.d_moy = 11 AND d.d_year = 2000" + + " GROUP BY d.d_year, i.i_brand, i.i_brand_id" + + " ORDER BY d.d_year, ext_price DESC, brand_id LIMIT 100"; + + // TPC-DS Q55: store ext sales price by brand for a manager/month/year. + static final String Q55 = + "SELECT i.i_brand_id brand_id, i.i_brand brand," + " SUM(ss.ss_ext_sales_price) ext_price" + + " FROM TPCDS.date_dim d, TPCDS.store_sales ss, TPCDS.item i" + + " WHERE d.d_date_sk = ss.ss_sold_date_sk AND ss.ss_item_sk = i.i_item_sk" + + " AND i.i_manager_id = 2 AND d.d_moy = 11 AND d.d_year = 2001" + + " GROUP BY i.i_brand, i.i_brand_id ORDER BY ext_price DESC, brand_id LIMIT 100"; + + // TPC-DS Q43: store sales by day of week (CASE pivot) for a region/year. + static final String Q43 = "SELECT s.s_store_name, s.s_store_id," + + " SUM(CASE WHEN d.d_day_name = 'Sunday' THEN ss.ss_sales_price ELSE 0 END) sun_sales," + + " SUM(CASE WHEN d.d_day_name = 'Monday' THEN ss.ss_sales_price ELSE 0 END) mon_sales," + + " SUM(CASE WHEN d.d_day_name = 'Tuesday' THEN ss.ss_sales_price ELSE 0 END) tue_sales," + + " SUM(CASE WHEN d.d_day_name = 'Wednesday' THEN ss.ss_sales_price ELSE 0 END) wed_sales," + + " SUM(CASE WHEN d.d_day_name = 'Thursday' THEN ss.ss_sales_price ELSE 0 END) thu_sales," + + " SUM(CASE WHEN d.d_day_name = 'Friday' THEN ss.ss_sales_price ELSE 0 END) fri_sales," + + " SUM(CASE WHEN d.d_day_name = 'Saturday' THEN ss.ss_sales_price ELSE 0 END) sat_sales" + + " FROM TPCDS.date_dim d, TPCDS.store_sales ss, TPCDS.store s" + + " WHERE d.d_date_sk = ss.ss_sold_date_sk AND ss.ss_store_sk = s.s_store_sk" + + " AND s.s_gmt_offset = -5 AND d.d_year = 2000" + " GROUP BY s.s_store_name, s.s_store_id" + + " ORDER BY s.s_store_name, s.s_store_id, sun_sales, mon_sales, tue_sales, wed_sales," + + " thu_sales, fri_sales, sat_sales LIMIT 100"; + + // TPC-DS Q96: count of store sales in an hour band / demographic / company. + static final String Q96 = "SELECT COUNT(*) cnt" + + " FROM TPCDS.store_sales ss, TPCDS.household_demographics hd, TPCDS.time_dim t," + + " TPCDS.store s" + + " WHERE ss.ss_sold_time_sk = t.t_time_sk AND ss.ss_hdemo_sk = hd.hd_demo_sk" + + " AND ss.ss_store_sk = s.s_store_sk AND t.t_hour >= 8 AND hd.hd_dep_count >= 0" + + " AND s.s_company_id = 1 ORDER BY cnt LIMIT 100"; + + // TPC-DS Q19: brand revenue where customer zip differs from store zip. + static final String Q19 = "SELECT i.i_brand_id brand_id, i.i_brand brand, i.i_manufact_id," + + " i.i_manufact, SUM(ss.ss_ext_sales_price) ext_price" + + " FROM TPCDS.date_dim d, TPCDS.store_sales ss, TPCDS.item i, TPCDS.customer c," + + " TPCDS.customer_address ca, TPCDS.store s" + + " WHERE d.d_date_sk = ss.ss_sold_date_sk AND ss.ss_item_sk = i.i_item_sk" + + " AND ss.ss_customer_sk = c.c_customer_sk AND c.c_current_addr_sk = ca.ca_address_sk" + + " AND ss.ss_store_sk = s.s_store_sk AND i.i_manager_id = 1 AND d.d_moy = 11" + + " AND d.d_year = 2000 AND SUBSTR(ca.ca_zip, 1, 5) <> SUBSTR(s.s_zip, 1, 5)" + + " GROUP BY i.i_brand, i.i_brand_id, i.i_manufact_id, i.i_manufact" + + " ORDER BY ext_price DESC, i.i_brand, i.i_brand_id, i.i_manufact_id, i.i_manufact LIMIT 100"; + + // TPC-DS Q46: coupon amount and profit by customer/city for a demographic segment. + static final String Q46 = "SELECT c.c_last_name, c.c_first_name, ca.ca_city," + + " SUM(ss.ss_coupon_amt) amt, SUM(ss.ss_net_profit) profit" + + " FROM TPCDS.store_sales ss, TPCDS.date_dim d, TPCDS.store s," + + " TPCDS.household_demographics hd, TPCDS.customer_address ca, TPCDS.customer c" + + " WHERE ss.ss_sold_date_sk = d.d_date_sk AND ss.ss_store_sk = s.s_store_sk" + + " AND ss.ss_hdemo_sk = hd.hd_demo_sk AND ss.ss_addr_sk = ca.ca_address_sk" + + " AND ss.ss_customer_sk = c.c_customer_sk" + + " AND (hd.hd_dep_count = 2 OR hd.hd_vehicle_count = 1) AND d.d_year = 2000" + + " AND s.s_city IN ('City0', 'City1', 'City2', 'City3', 'City4')" + + " GROUP BY c.c_last_name, c.c_first_name, ca.ca_city" + + " ORDER BY c.c_last_name, c.c_first_name, ca.ca_city, amt, profit LIMIT 100"; + + // TPC-DS Q50: store return lag buckets by store (self-join on date_dim). + static final String Q50 = "SELECT s.s_store_name, s.s_store_id," + + " SUM(CASE WHEN (d2.d_date_sk - d1.d_date_sk <= 1) THEN 1 ELSE 0 END) days_le_1," + + " SUM(CASE WHEN (d2.d_date_sk - d1.d_date_sk > 1 AND d2.d_date_sk - d1.d_date_sk <= 3)" + + " THEN 1 ELSE 0 END) days_2_3," + + " SUM(CASE WHEN (d2.d_date_sk - d1.d_date_sk > 3) THEN 1 ELSE 0 END) days_gt_3" + + " FROM TPCDS.store_sales ss, TPCDS.store_returns sr, TPCDS.date_dim d1," + + " TPCDS.date_dim d2, TPCDS.store s" + + " WHERE ss.ss_ticket_number = sr.sr_ticket_number AND ss.ss_item_sk = sr.sr_item_sk" + + " AND ss.ss_sold_date_sk = d1.d_date_sk AND sr.sr_returned_date_sk = d2.d_date_sk" + + " AND ss.ss_store_sk = s.s_store_sk AND d2.d_year = 2000" + + " GROUP BY s.s_store_name, s.s_store_id ORDER BY s.s_store_name, s.s_store_id LIMIT 100"; + + // TPC-DS Q73: tickets per customer via a grouped derived table joined to customer. + static final String Q73 = "SELECT c.c_last_name, c.c_first_name, dj.ss_ticket_number, dj.cnt" + + " FROM (SELECT ss.ss_ticket_number ss_ticket_number, ss.ss_customer_sk ss_customer_sk," + + " COUNT(*) cnt FROM TPCDS.store_sales ss, TPCDS.date_dim d, TPCDS.store s," + + " TPCDS.household_demographics hd" + + " WHERE ss.ss_sold_date_sk = d.d_date_sk AND ss.ss_store_sk = s.s_store_sk" + + " AND ss.ss_hdemo_sk = hd.hd_demo_sk AND d.d_year = 2000 AND hd.hd_vehicle_count >= 0" + + " AND s.s_company_id = 1 GROUP BY ss.ss_ticket_number, ss.ss_customer_sk) dj," + + " TPCDS.customer c" + " WHERE dj.ss_customer_sk = c.c_customer_sk AND dj.cnt BETWEEN 1 AND 5" + + " ORDER BY dj.cnt DESC, c.c_last_name, c.c_first_name, dj.ss_ticket_number LIMIT 20"; + + // TPC-DS Q82: items in a price range with on-hand inventory that were also sold in store. + static final String Q82 = "SELECT i.i_item_id, i.i_item_desc, i.i_current_price" + + " FROM TPCDS.item i, TPCDS.inventory inv, TPCDS.date_dim d, TPCDS.store_sales ss" + + " WHERE i.i_current_price BETWEEN 20 AND 80 AND inv.inv_item_sk = i.i_item_sk" + + " AND d.d_date_sk = inv.inv_date_sk AND inv.inv_quantity_on_hand BETWEEN 100 AND 500" + + " AND i.i_manufact_id IN (1, 2, 3, 4, 5) AND inv.inv_item_sk = ss.ss_item_sk" + + " GROUP BY i.i_item_id, i.i_item_desc, i.i_current_price ORDER BY i.i_item_id LIMIT 100"; + + // TPC-DS Q21: inventory before/after a pivot date by warehouse/item (CASE pivot). + static final String Q21 = "SELECT w.w_warehouse_name, i.i_item_id," + + " SUM(CASE WHEN d.d_date_sk < 12 THEN inv.inv_quantity_on_hand ELSE 0 END) inv_before," + + " SUM(CASE WHEN d.d_date_sk >= 12 THEN inv.inv_quantity_on_hand ELSE 0 END) inv_after" + + " FROM TPCDS.inventory inv, TPCDS.warehouse w, TPCDS.item i, TPCDS.date_dim d" + + " WHERE i.i_item_sk = inv.inv_item_sk AND inv.inv_warehouse_sk = w.w_warehouse_sk" + + " AND inv.inv_date_sk = d.d_date_sk AND i.i_current_price BETWEEN 10 AND 100" + + " AND i.i_manufact_id IN (1, 2, 3) AND w.w_warehouse_sk IN (1, 2)" + + " GROUP BY w.w_warehouse_name, i.i_item_id" + + " ORDER BY w.w_warehouse_name, i.i_item_id LIMIT 100"; + + // TPC-DS Q26: average catalog sale measures by item for a gender/marital/promo segment. + static final String Q26 = "SELECT i.i_item_id, AVG(cs.cs_quantity) agg1," + + " AVG(cs.cs_list_price) agg2, AVG(cs.cs_coupon_amt) agg3, AVG(cs.cs_sales_price) agg4" + + " FROM TPCDS.catalog_sales cs, TPCDS.customer_demographics cd, TPCDS.date_dim d," + + " TPCDS.item i, TPCDS.promotion p" + + " WHERE cs.cs_sold_date_sk = d.d_date_sk AND cs.cs_item_sk = i.i_item_sk" + + " AND cs.cs_bill_cdemo_sk = cd.cd_demo_sk AND cs.cs_promo_sk = p.p_promo_sk" + + " AND cd.cd_gender = 'F' AND cd.cd_marital_status = 'M'" + + " AND (p.p_channel_email = 'N' OR p.p_channel_event = 'N') AND d.d_year = 2000" + + " GROUP BY i.i_item_id ORDER BY i.i_item_id LIMIT 100"; + + private static final String[][] Q03_EXPECTED = + { { "2000", "1002", "brand#1002", "41" }, { "2000", "1000", "brand#1000", "37" }, + { "2000", "1005", "brand#1005", "36" }, { "2000", "1007", "brand#1007", "18" }, + { "2000", "1004", "brand#1004", "5" }, { "2001", "1004", "brand#1004", "36" }, + { "2001", "1002", "brand#1002", "21" }, { "2001", "1007", "brand#1007", "18" }, + { "2001", "1000", "brand#1000", "11" }, { "2001", "1005", "brand#1005", "7" }, }; + private static final String[][] Q07_EXPECTED = { { "ITEM0000000001", "3", "73.5", "16", "57.5" }, + { "ITEM0000000003", "13", "30.5", "2", "27.5" }, { "ITEM0000000005", "7", "37.5", "8", "26.5" }, + { "ITEM0000000007", "8", "72.5", "1", "67.5" }, + { "ITEM0000000009", "14", "92.5", "10", "81.5" }, + { "ITEM0000000011", "2", "106.5", "15", "98.5" }, + { "ITEM0000000013", "5", "66.5", "10", "55.5" }, { "ITEM0000000015", "9", "34.5", "2", "15.5" }, + { "ITEM0000000017", "8", "51.5", "14", "49.5" }, }; + private static final String[][] Q42_EXPECTED = { { "2000", "5", "Jewelry", "748" }, + { "2000", "3", "Electronics", "591" }, { "2000", "1", "Books", "61" }, }; + private static final String[][] Q52_EXPECTED = { { "2000", "1002", "brand#1002", "748" }, + { "2000", "1004", "brand#1004", "591" }, { "2000", "1000", "brand#1000", "61" }, }; + private static final String[][] Q55_EXPECTED = { { "1001", "brand#1001", "456.5" }, + { "1003", "brand#1003", "257.5" }, { "1005", "brand#1005", "82.5" }, }; + private static final String[][] Q43_EXPECTED = + { { "Store 1", "STORE0000000001", "0", "0", "0", "0", "1638", "0", "0" }, + { "Store 5", "STORE0000000005", "1229", "0", "0", "0", "0", "0", "0" }, }; + private static final String[][] Q96_EXPECTED = { { "192" }, }; + private static final String[][] Q19_EXPECTED = + { { "1002", "brand#1002", "1", "manufact#1", "748" }, + { "1004", "brand#1004", "1", "manufact#1", "591" }, + { "1000", "brand#1000", "1", "manufact#1", "61" }, }; + private static final String[][] Q46_EXPECTED = { { "Last10", "First10", "City2", "45", "2338.5" }, + { "Last14", "First14", "City6", "48", "2291.5" }, + { "Last15", "First15", "City0", "29", "2357" }, { "Last18", "First18", "City3", "31", "2724" }, + { "Last2", "First2", "City1", "24", "632" }, { "Last20", "First20", "City5", "45", "3686" }, + { "Last22", "First22", "City1", "35", "2991" }, { "Last26", "First26", "City5", "57", "3988" }, + { "Last27", "First27", "City6", "62", "1133" }, { "Last3", "First3", "City2", "26", "908.5" }, + { "Last30", "First30", "City2", "27", "542" }, { "Last6", "First6", "City5", "42", "1937.5" }, + { "Last8", "First8", "City0", "24", "701.5" }, }; + private static final String[][] Q50_EXPECTED = { { "Store 1", "STORE0000000001", "24", "0", "0" }, + { "Store 3", "STORE0000000003", "24", "0", "0" }, + { "Store 5", "STORE0000000005", "24", "0", "0" }, }; + private static final String[][] Q73_EXPECTED = + { { "Last1", "First1", "222", "1" }, { "Last1", "First1", "245", "1" }, + { "Last1", "First1", "268", "1" }, { "Last10", "First10", "16", "1" }, + { "Last10", "First10", "39", "1" }, { "Last10", "First10", "62", "1" }, + { "Last10", "First10", "85", "1" }, { "Last11", "First11", "5", "1" }, + { "Last11", "First11", "28", "1" }, { "Last11", "First11", "51", "1" }, + { "Last11", "First11", "74", "1" }, { "Last11", "First11", "97", "1" }, + { "Last12", "First12", "17", "1" }, { "Last12", "First12", "40", "1" }, + { "Last12", "First12", "63", "1" }, { "Last12", "First12", "86", "1" }, + { "Last12", "First12", "109", "1" }, { "Last13", "First13", "6", "1" }, + { "Last13", "First13", "29", "1" }, { "Last13", "First13", "52", "1" }, }; + private static final String[][] Q82_EXPECTED = + { { "ITEM0000000010", "Item description 10", "20.99" }, + { "ITEM0000000011", "Item description 11", "21.99" }, + { "ITEM0000000012", "Item description 12", "22.99" }, + { "ITEM0000000013", "Item description 13", "23.99" }, + { "ITEM0000000014", "Item description 14", "24.99" }, + { "ITEM0000000015", "Item description 15", "25.99" }, + { "ITEM0000000016", "Item description 16", "26.99" }, + { "ITEM0000000017", "Item description 17", "27.99" }, + { "ITEM0000000018", "Item description 18", "28.99" }, + { "ITEM0000000019", "Item description 19", "29.99" }, + { "ITEM0000000020", "Item description 20", "30.99" }, + { "ITEM0000000021", "Item description 21", "31.99" }, + { "ITEM0000000022", "Item description 22", "32.99" }, + { "ITEM0000000023", "Item description 23", "33.99" }, + { "ITEM0000000024", "Item description 24", "34.99" }, }; + private static final String[][] Q21_EXPECTED = + { { "Warehouse 1", "ITEM0000000001", "1091", "1203" }, + { "Warehouse 1", "ITEM0000000002", "1053", "1257" }, + { "Warehouse 1", "ITEM0000000003", "1681", "1517" }, + { "Warehouse 1", "ITEM0000000006", "1631", "1011" }, + { "Warehouse 1", "ITEM0000000007", "1237", "1972" }, + { "Warehouse 1", "ITEM0000000008", "1187", "1271" }, + { "Warehouse 1", "ITEM0000000011", "1306", "1307" }, + { "Warehouse 1", "ITEM0000000012", "587", "1511" }, + { "Warehouse 1", "ITEM0000000013", "1212", "1372" }, + { "Warehouse 1", "ITEM0000000016", "1611", "1589" }, + { "Warehouse 1", "ITEM0000000017", "1242", "1589" }, + { "Warehouse 1", "ITEM0000000018", "1012", "1618" }, + { "Warehouse 1", "ITEM0000000021", "1582", "948" }, + { "Warehouse 1", "ITEM0000000022", "1074", "1872" }, + { "Warehouse 1", "ITEM0000000023", "1792", "1537" }, + { "Warehouse 2", "ITEM0000000001", "831", "1683" }, + { "Warehouse 2", "ITEM0000000002", "1462", "899" }, + { "Warehouse 2", "ITEM0000000003", "1873", "1837" }, + { "Warehouse 2", "ITEM0000000006", "1010", "1437" }, + { "Warehouse 2", "ITEM0000000007", "1107", "1515" }, + { "Warehouse 2", "ITEM0000000008", "1255", "1714" }, + { "Warehouse 2", "ITEM0000000011", "1073", "941" }, + { "Warehouse 2", "ITEM0000000012", "1936", "1490" }, + { "Warehouse 2", "ITEM0000000013", "1015", "2189" }, + { "Warehouse 2", "ITEM0000000016", "1331", "1388" }, + { "Warehouse 2", "ITEM0000000017", "1721", "1268" }, + { "Warehouse 2", "ITEM0000000018", "1434", "900" }, + { "Warehouse 2", "ITEM0000000021", "1306", "1858" }, + { "Warehouse 2", "ITEM0000000022", "946", "1806" }, + { "Warehouse 2", "ITEM0000000023", "1159", "1383" }, }; + private static final String[][] Q26_EXPECTED = { { "ITEM0000000003", "2", "109.5", "13", "92.5" }, + { "ITEM0000000006", "8", "77.5", "6", "68.5" }, + { "ITEM0000000009", "11", "76.5", "18", "57.5" }, { "ITEM0000000012", "16", "63.5", "9", "54" }, + { "ITEM0000000015", "6", "32.5", "6", "18.5" }, + { "ITEM0000000018", "18", "62.5", "11", "52.5" }, + { "ITEM0000000021", "18", "9.5", "13", "5.5" }, + { "ITEM0000000024", "17", "77.5", "12", "67.5" }, }; + + /** Label for {@link TPCDSLikeExpectedRegenerator}. */ + public static Map queries() { + Map q = new LinkedHashMap<>(); + q.put("Q03", Q03); + q.put("Q07", Q07); + q.put("Q42", Q42); + q.put("Q52", Q52); + q.put("Q55", Q55); + q.put("Q43", Q43); + q.put("Q96", Q96); + q.put("Q19", Q19); + q.put("Q46", Q46); + q.put("Q50", Q50); + q.put("Q73", Q73); + q.put("Q82", Q82); + q.put("Q21", Q21); + q.put("Q26", Q26); + return q; + } + + @Test + public void testQ03() throws SQLException { + check("Q03", Q03, Q03_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "SS_I" }); + } + + @Test + public void testQ07() throws SQLException { + check("Q07", Q07, Q07_EXPECTED, "HASH BUILD"); + } + + @Test + public void testQ42() throws SQLException { + check("Q42", Q42, Q42_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "SS_I" }); + } + + @Test + public void testQ52() throws SQLException { + check("Q52", Q52, Q52_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "SS_I" }); + } + + @Test + public void testQ55() throws SQLException { + check("Q55", Q55, Q55_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "SS_I" }); + } + + @Test + public void testQ43() throws SQLException { + check("Q43", Q43, Q43_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "SS_I" }); + } + + @Test + public void testQ96() throws SQLException { + check("Q96", Q96, Q96_EXPECTED, "HASH BUILD"); + } + + @Test + public void testQ19() throws SQLException { + check("Q19", Q19, Q19_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "SS_I" }); + } + + @Test + public void testQ46() throws SQLException { + check("Q46", Q46, Q46_EXPECTED, "HASH BUILD"); + } + + @Test + public void testQ50() throws SQLException { + check("Q50", Q50, Q50_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "SS_I" }); + } + + @Test + public void testQ73() throws SQLException { + check("Q73", Q73, Q73_EXPECTED, "HASH BUILD"); + } + + @Test + public void testQ82() throws SQLException { + check("Q82", Q82, Q82_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "INV_I" }); + } + + @Test + public void testQ21() throws SQLException { + check("Q21", Q21, Q21_EXPECTED, new String[] { "HASH BUILD" }, new String[] { "INV_I" }); + } + + @Test + public void testQ26() throws SQLException { + check("Q26", Q26, Q26_EXPECTED, "HASH BUILD"); + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/HBaseScanMetricsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/HBaseScanMetricsIT.java index a8f86f7b277..354b72ab82d 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/HBaseScanMetricsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/HBaseScanMetricsIT.java @@ -17,6 +17,8 @@ */ package org.apache.phoenix.monitoring; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -33,11 +35,8 @@ import org.apache.hadoop.hbase.regionserver.CompactSplit; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.util.VersionInfo; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.hbase.index.IndexRegionObserver; -import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PhoenixRuntime; @@ -285,11 +284,7 @@ public void testQueryOnUncoveredIndex() throws Exception { stmt.execute("UPSERT INTO " + tableName + " (k1, k2, v1, v2) VALUES (3, 'c', 'c1', 'c2')"); conn.commit(); String sql = "SELECT k1, k2, v1, v2 FROM " + tableName + " WHERE v1 = 'b1'"; - ExplainPlan explainPlan = - stmt.unwrap(PhoenixStatement.class).optimizeQuery(sql).getExplainPlan(); - ExplainPlanAttributes planAttributes = explainPlan.getPlanStepsAsAttributes(); - String tableNameFromExplainPlan = planAttributes.getTableName(); - Assert.assertEquals(indexName, tableNameFromExplainPlan); + assertPlan(conn, sql).table(indexName); ResultSet rs = stmt.executeQuery(sql); assertOnReadsFromMemstore(indexName, getQueryReadMetrics(rs)); TestUtil.flush(utility, TableName.valueOf(tableName)); @@ -316,11 +311,7 @@ public void testQueryOnCoveredIndexWithoutReadRepair() throws Exception { stmt.execute("UPSERT INTO " + tableName + " (k1, k2, v1, v2) VALUES (3, 'c', 'c1', 'c2')"); conn.commit(); String sql = "SELECT k1, k2, v1, v2 FROM " + tableName + " WHERE v1 = 'b1'"; - ExplainPlan explainPlan = - stmt.unwrap(PhoenixStatement.class).optimizeQuery(sql).getExplainPlan(); - ExplainPlanAttributes planAttributes = explainPlan.getPlanStepsAsAttributes(); - String tableNameFromExplainPlan = planAttributes.getTableName(); - Assert.assertEquals(indexName, tableNameFromExplainPlan); + assertPlan(conn, sql).table(indexName); ResultSet rs = stmt.executeQuery(sql); assertOnReadsFromMemstore(indexName, getQueryReadMetrics(rs)); TestUtil.flush(utility, TableName.valueOf(tableName)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java index ba4c06457b1..6385dc4980e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.rpc; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -32,9 +33,9 @@ import org.apache.hadoop.hbase.ipc.CallRunner; import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.junit.AfterClass; @@ -102,12 +103,11 @@ public void testIndexQos() throws Exception { stmt.setString(1, "v1"); // verify that the query does a range scan on the index table - ResultSet rs = stmt.executeQuery("EXPLAIN " + selectSql); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexFullName + " ['v1']", - QueryUtil.getExplainPlan(rs)); + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).scanType("RANGE SCAN") + .tableContains(indexFullName).keyRanges("['v1']"); // verify that the correct results are returned - rs = stmt.executeQuery(); + ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); assertEquals("k1", rs.getString(1)); assertEquals("v2", rs.getString(2)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixServerRpcIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixServerRpcIT.java index 3dc82388552..4c2ed1e2ee1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixServerRpcIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixServerRpcIT.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.rpc; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -45,10 +46,10 @@ import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.junit.After; @@ -115,12 +116,11 @@ public void testIndexQos() throws Exception { stmt.setString(1, "v1"); // verify that the query does a range scan on the index table - ResultSet rs = stmt.executeQuery("EXPLAIN " + selectSql); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableFullName + " ['v1']", - QueryUtil.getExplainPlan(rs)); + assertPlan(stmt.unwrap(PhoenixPreparedStatement.class)).scanType("RANGE SCAN") + .tableContains(indexTableFullName).keyRanges("['v1']"); // verify that the correct results are returned - rs = stmt.executeQuery(); + ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); assertEquals("k1", rs.getString(1)); assertEquals("v2", rs.getString(2)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/ConditionalTTLExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/ConditionalTTLExpressionIT.java index 3cf200e46e5..679cc2c92fe 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/schema/ConditionalTTLExpressionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/ConditionalTTLExpressionIT.java @@ -27,6 +27,7 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REPAIR_EXTRA_VERIFIED_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER; import static org.apache.phoenix.util.TestUtil.retainSingleQuotes; import static org.junit.Assert.assertEquals; @@ -71,7 +72,6 @@ import org.apache.phoenix.hbase.index.IndexRegionObserver; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.jdbc.PhoenixConnection; -import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.mapreduce.index.IndexTool; import org.apache.phoenix.query.PhoenixTestBuilder; import org.apache.phoenix.query.PhoenixTestBuilder.SchemaBuilder; @@ -83,7 +83,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.ManualEnvironmentEdge; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -1037,9 +1036,7 @@ public void testUnverifiedRows() throws Exception { String dql = String.format("select VAL2, VAL5 from %s where VAL1='%s' AND ID2=0", fullDataTableName, val1_0); try (ResultSet rs1 = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs1.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(fullIndexName)); + assertPlan(conn, dql).tableContains(fullIndexName); assertTrue(rs1.next()); assertEquals(rs1.getInt("VAL2"), val2); assertFalse(rs1.getBoolean(ttlCol)); @@ -1048,9 +1045,7 @@ public void testUnverifiedRows() throws Exception { dql = String.format("select VAL2, VAL5 from %s where VAL1='%s' AND ID2=1", fullDataTableName, val1_1); try (ResultSet rs1 = conn.createStatement().executeQuery(dql)) { - PhoenixResultSet prs = rs1.unwrap(PhoenixResultSet.class); - String explainPlan = QueryUtil.getExplainPlan(prs.getUnderlyingIterator()); - assertTrue(explainPlan.contains(fullIndexName)); + assertPlan(conn, dql).tableContains(fullIndexName); assertNotEquals(isStrictTTL, rs1.next()); } // run the reverse index verification tool diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/BaseStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/BaseStatsCollectorIT.java index 6622f208063..f6771376b4a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/BaseStatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/BaseStatsCollectorIT.java @@ -21,12 +21,12 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE; import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.MAPREDUCE_JOB_TYPE; import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.MRJobType.UPDATE_STATS; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.getAllSplits; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -54,12 +54,9 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.mapreduce.Job; -import org.apache.phoenix.compile.ExplainPlan; -import org.apache.phoenix.compile.ExplainPlanAttributes; import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; -import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.KeyRange; @@ -71,7 +68,6 @@ import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature; import org.apache.phoenix.transaction.TransactionFactory; import org.apache.phoenix.util.MetaDataUtil; -import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; @@ -258,17 +254,9 @@ public void testUpdateEmptyStats() throws Exception { conn.createStatement() .execute("CREATE TABLE " + fullTableName + " ( k CHAR(1) PRIMARY KEY )" + tableDDLOptions); collectStatistics(conn, fullTableName); - ExplainPlan plan = conn.prepareStatement("SELECT * FROM " + fullTableName) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(1, (int) planAttributes.getSplitsChunk()); - assertEquals(0, (long) planAttributes.getEstimatedRows()); - assertEquals(20, (long) planAttributes.getEstimatedSizeInBytes()); - assertEquals("PARALLEL 1-WAY", planAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", planAttributes.getExplainScanType()); - assertEquals(physicalTableName, planAttributes.getTableName()); - assertEquals("SERVER FILTER BY " + (columnEncoded ? "FIRST KEY ONLY" : "EMPTY COLUMN ONLY"), - planAttributes.getServerWhereFilter()); + assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(1).scanEstimatedRows(0L) + .scanEstimatedBytes(20L).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(physicalTableName).serverProjectionFilter(columnEncoded); conn.close(); } @@ -286,45 +274,24 @@ public void testSomeUpdateEmptyStats() throws Exception { // if we are using the ONE_CELL_PER_COLUMN_FAMILY storage scheme, we will have the single kv // even though there are no values for col family v2 - ExplainPlan plan = conn.prepareStatement("SELECT v2 FROM " + fullTableName + " WHERE v2='foo'") - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(columnEncoded && !mutable ? 4 : 3, (int) planAttributes.getSplitsChunk()); - assertEquals(columnEncoded && !mutable ? 1 : 0, (long) planAttributes.getEstimatedRows()); - assertEquals(columnEncoded && !mutable ? 38 : 20, - (long) planAttributes.getEstimatedSizeInBytes()); - assertEquals("PARALLEL 3-WAY", planAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", planAttributes.getExplainScanType()); - assertEquals(physicalTableName, planAttributes.getTableName()); - assertEquals("SERVER FILTER BY B.V2 = 'foo'", planAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", planAttributes.getClientSortAlgo()); + assertPlan(conn, "SELECT v2 FROM " + fullTableName + " WHERE v2='foo'") + .splitsChunk(columnEncoded && !mutable ? 4 : 3) + .scanEstimatedRows(columnEncoded && !mutable ? 1L : 0L) + .scanEstimatedBytes(columnEncoded && !mutable ? 38L : 20L).iteratorType("PARALLEL 3-WAY") + .scanType("FULL SCAN").table(physicalTableName) + .serverWhereFilter("SERVER FILTER BY B.V2 = 'foo'").clientSortAlgo("CLIENT MERGE SORT"); long estimatedSizeInBytes = columnEncoded ? 28 : TransactionFactory.Provider.OMID.name().equals(transactionProvider) ? 38 : 34; - plan = conn.prepareStatement("SELECT * FROM " + fullTableName) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(4, (int) planAttributes.getSplitsChunk()); - assertEquals(1, (long) planAttributes.getEstimatedRows()); - assertEquals(estimatedSizeInBytes, (long) planAttributes.getEstimatedSizeInBytes()); - assertEquals("PARALLEL 3-WAY", planAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", planAttributes.getExplainScanType()); - assertEquals(physicalTableName, planAttributes.getTableName()); - assertNull(planAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", planAttributes.getClientSortAlgo()); - - plan = conn.prepareStatement("SELECT * FROM " + fullTableName + " WHERE k = 'a'") - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(1, (int) planAttributes.getSplitsChunk()); - assertEquals(1, (long) planAttributes.getEstimatedRows()); - assertEquals(columnEncoded ? 204 : 202, (long) planAttributes.getEstimatedSizeInBytes()); - assertEquals("PARALLEL 1-WAY", planAttributes.getIteratorTypeAndScanSize()); - assertEquals("POINT LOOKUP ON 1 KEY ", planAttributes.getExplainScanType()); - assertEquals(physicalTableName, planAttributes.getTableName()); - assertNull(planAttributes.getServerWhereFilter()); - assertEquals("CLIENT MERGE SORT", planAttributes.getClientSortAlgo()); + assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(4).scanEstimatedRows(1L) + .scanEstimatedBytes(estimatedSizeInBytes).iteratorType("PARALLEL 3-WAY").scanType("FULL SCAN") + .table(physicalTableName).serverWhereFilter(null).clientSortAlgo("CLIENT MERGE SORT"); + + assertPlan(conn, "SELECT * FROM " + fullTableName + " WHERE k = 'a'").splitsChunk(1) + .scanEstimatedRows(1L).scanEstimatedBytes(columnEncoded ? 204L : 202L) + .iteratorType("PARALLEL 1-WAY").scanType("POINT LOOKUP ON 1 KEY").table(physicalTableName) + .serverWhereFilter(null).clientSortAlgo("CLIENT MERGE SORT"); conn.close(); } @@ -333,7 +300,6 @@ public void testSomeUpdateEmptyStats() throws Exception { public void testUpdateStats() throws Exception { Connection conn; PreparedStatement stmt; - ResultSet rs; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); conn = getConnection(); conn.createStatement().execute("CREATE TABLE " + fullTableName @@ -343,9 +309,8 @@ public void testUpdateStats() throws Exception { Array array; conn = upsertValues(props, fullTableName); collectStatistics(conn, fullTableName); - rs = conn.createStatement().executeQuery("EXPLAIN SELECT k FROM " + fullTableName); - rs.next(); - long rows1 = (Long) rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_ROWS_READ_COLUMN); + Long rows1 = + assertPlan(conn, "SELECT k FROM " + fullTableName).attributes().getScanEstimatedRows(); stmt = upsertStmt(conn, fullTableName); stmt.setString(1, "z"); s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" }; @@ -357,9 +322,8 @@ public void testUpdateStats() throws Exception { stmt.execute(); conn.commit(); collectStatistics(conn, fullTableName); - rs = conn.createStatement().executeQuery("EXPLAIN SELECT k FROM " + fullTableName); - rs.next(); - long rows2 = (Long) rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_ROWS_READ_COLUMN); + Long rows2 = + assertPlan(conn, "SELECT k FROM " + fullTableName).attributes().getScanEstimatedRows(); assertNotEquals(rows1, rows2); conn.close(); } @@ -611,15 +575,9 @@ public void testWithMultiCF() throws Exception { : (TransactionFactory.Provider.OMID.name().equals(transactionProvider)) ? 25044 : 12420; - ExplainPlan plan = conn.prepareStatement("SELECT * FROM " + fullTableName) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - ExplainPlanAttributes planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(26, (int) planAttributes.getSplitsChunk()); - assertEquals(25, (long) planAttributes.getEstimatedRows()); - assertEquals(sizeInBytes, (long) planAttributes.getEstimatedSizeInBytes()); - assertEquals("PARALLEL 1-WAY", planAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", planAttributes.getExplainScanType()); - assertEquals(physicalTableName, planAttributes.getTableName()); + assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(26).scanEstimatedRows(25L) + .scanEstimatedBytes(sizeInBytes).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(physicalTableName); ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices(); List regions = @@ -680,15 +638,9 @@ public void testWithMultiCF() throws Exception { assertEquals(0, rs.getLong(1)); assertFalse(rs.next()); - plan = conn.prepareStatement("SELECT * FROM " + fullTableName) - .unwrap(PhoenixPreparedStatement.class).optimizeQuery().getExplainPlan(); - planAttributes = plan.getPlanStepsAsAttributes(); - assertEquals(1, (int) planAttributes.getSplitsChunk()); - assertNull(planAttributes.getEstimatedRows()); - assertNull(planAttributes.getEstimatedSizeInBytes()); - assertEquals("PARALLEL 1-WAY", planAttributes.getIteratorTypeAndScanSize()); - assertEquals("FULL SCAN ", planAttributes.getExplainScanType()); - assertEquals(physicalTableName, planAttributes.getTableName()); + assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(1).scanEstimatedRows(null) + .scanEstimatedBytes(null).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + .table(physicalTableName); } @Test diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java index beed114ef0f..097ea96f4c0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java @@ -175,7 +175,7 @@ public void testCreateLocalIndexStatement() throws Exception { String createIndexStatement = "CREATE LOCAL INDEX " + indexName + " ON " + pTableFullName + "(v1 DESC, k) INCLUDE (v2)"; String createIndexStatement2 = "CREATE LOCAL INDEX " + indexName2 + " ON " + pTableFullName - + "( LPAD(v1,10) DESC, k) INCLUDE (v2)"; + + "(LPAD(v1,10) DESC, k) INCLUDE (v2)"; List queries = new ArrayList() { }; @@ -203,7 +203,7 @@ public void testCreateLocalIndexStatementLowerCase() throws Exception { String createTableStatement = "CREATE TABLE " + pTableFullName + "(K VARCHAR NOT NULL PRIMARY KEY, \"v1\" VARCHAR, V2 VARCHAR)" + properties; String createIndexStatement = "CREATE LOCAL INDEX " + indexName + " ON " + pTableFullName - + "( LPAD(\"v1\",10) DESC, K) INCLUDE (V2)"; + + "(LPAD(\"v1\",10) DESC, K) INCLUDE (V2)"; List queries = new ArrayList() { }; diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/JoinQueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/JoinQueryCompilerTest.java index c6fe30a4463..a036cd1c581 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/JoinQueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/JoinQueryCompilerTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.compile; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.JOIN_CUSTOMER_TABLE_DISPLAY_NAME; import static org.apache.phoenix.util.TestUtil.JOIN_CUSTOMER_TABLE_FULL_NAME; import static org.apache.phoenix.util.TestUtil.JOIN_ITEM_TABLE_DISPLAY_NAME; @@ -30,13 +31,11 @@ import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import org.apache.phoenix.compile.JoinCompiler.JoinTable; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.BaseConnectionlessQueryTest; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.TestUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -75,22 +74,25 @@ public static synchronized void createJoinTables() throws SQLException { public void testExplainPlan() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); String query = - "EXPLAIN SELECT s.\"supplier_id\", \"order_id\", c.name, i.name, quantity, o.\"date\" FROM " + "SELECT s.\"supplier_id\", \"order_id\", c.name, i.name, quantity, o.\"date\" FROM " + JOIN_ORDER_TABLE_FULL_NAME + " o LEFT JOIN " + JOIN_CUSTOMER_TABLE_FULL_NAME + " c ON o.\"customer_id\" = c.\"customer_id\" AND c.name LIKE 'C%' LEFT JOIN " + JOIN_ITEM_TABLE_FULL_NAME + " i ON o.\"item_id\" = i.\"item_id\" RIGHT JOIN " + JOIN_SUPPLIER_TABLE_FULL_NAME + " s ON s.\"supplier_id\" = i.\"supplier_id\" WHERE i.name LIKE 'T%'"; - ResultSet rs = conn.createStatement().executeQuery(query); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_SUPPLIER_TABLE_DISPLAY_NAME + "\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ORDER_TABLE_DISPLAY_NAME + "\n" - + " PARALLEL LEFT-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_CUSTOMER_TABLE_DISPLAY_NAME - + "\n" + " SERVER FILTER BY NAME LIKE 'C%'\n" - + " PARALLEL LEFT-JOIN TABLE 1\n" - + " CLIENT PARALLEL 1-WAY FULL SCAN OVER " + JOIN_ITEM_TABLE_DISPLAY_NAME - + "\n" + " AFTER-JOIN SERVER FILTER BY I.NAME LIKE 'T%'", QueryUtil.getExplainPlan(rs)); + // RIGHT JOIN drives the scan over SUPPLIER, with the rest of the join tree nested as sub-plans. + // The outer join is swapped to build the left input (HASH BUILD LEFT). The nested left-join + // tree builds its right inputs (HASH BUILD RIGHT). + assertPlan(conn, query).scanType("FULL SCAN").table(JOIN_SUPPLIER_TABLE_DISPLAY_NAME) + .serverFirstKeyOnlyProjection(true) + .afterJoinFilter("AFTER-JOIN SERVER FILTER BY I.NAME LIKE 'T%'").subPlanCount(1).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD LEFT */") + .scanType("FULL SCAN").table(JOIN_ORDER_TABLE_DISPLAY_NAME).subPlanCount(2).subPlan(0) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(JOIN_CUSTOMER_TABLE_DISPLAY_NAME) + .serverWhereFilter("SERVER FILTER BY NAME LIKE 'C%'").end().subPlan(1) + .abstractExplainPlan("PARALLEL LEFT-JOIN TABLE 1 /* HASH BUILD RIGHT */") + .scanType("FULL SCAN").table(JOIN_ITEM_TABLE_DISPLAY_NAME).end().end(); } @Test diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java index 59bdac5ff62..b0c7882ec95 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java @@ -19,6 +19,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.assertDegenerate; import static org.junit.Assert.assertArrayEquals; @@ -83,6 +84,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.parse.ParseNodeFactory; import org.apache.phoenix.query.BaseConnectionlessQueryTest; import org.apache.phoenix.query.QueryConstants; @@ -105,7 +107,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -1348,13 +1349,6 @@ public void testDuplicateKVColumn() throws Exception { } } - private void assertImmutableRows(Connection conn, String fullTableName, boolean expectedValue) - throws SQLException { - PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - assertEquals(expectedValue, - pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)).isImmutableRows()); - } - @Test public void testInvalidNegativeArrayIndex() throws Exception { String query = "SELECT a_double_array[-20] FROM table_with_array"; @@ -1597,7 +1591,6 @@ public void testGroupByLimitOptimization() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); conn.createStatement().execute( "CREATE TABLE t (k1 varchar, k2 varchar, v varchar, constraint pk primary key(k1,k2))"); - ResultSet rs; String[] queries = { "SELECT DISTINCT v FROM T LIMIT 3", "SELECT v FROM T GROUP BY v,k1 LIMIT 3", "SELECT count(*) FROM T GROUP BY k1 LIMIT 3", "SELECT max(v) FROM T GROUP BY k1,k2 LIMIT 3", "SELECT k1,k2 FROM T GROUP BY k1,k2 LIMIT 3", @@ -1605,12 +1598,8 @@ public void testGroupByLimitOptimization() throws Exception { // of GROUP BY key not // important }; - String query; - for (int i = 0; i < queries.length; i++) { - query = queries[i]; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - assertTrue("Expected to find GROUP BY limit optimization in: " + query, - QueryUtil.getExplainPlan(rs).contains(" LIMIT 3 GROUPS")); + for (String query : queries) { + assertPlan(conn, query).serverGroupByLimit(3); } } @@ -1619,7 +1608,6 @@ public void testNoGroupByLimitOptimization() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); conn.createStatement().execute( "CREATE TABLE t (k1 varchar, k2 varchar, v varchar, constraint pk primary key(k1,k2))"); - ResultSet rs; String[] queries = { // "SELECT DISTINCT v FROM T ORDER BY v LIMIT 3", // "SELECT v FROM T GROUP BY v,k1 ORDER BY v LIMIT 3", @@ -1627,13 +1615,8 @@ public void testNoGroupByLimitOptimization() throws Exception { "SELECT count(1) FROM T GROUP BY v,k1 LIMIT 3", "SELECT max(v) FROM T GROUP BY k1,k2 HAVING count(k1) > 1 LIMIT 3", "SELECT count(v) FROM T GROUP BY to_date(k2),k1 LIMIT 3", }; - String query; - for (int i = 0; i < queries.length; i++) { - query = queries[i]; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertFalse("Did not expected to find GROUP BY limit optimization in: " + query, - explainPlan.contains(" LIMIT 3 GROUPS")); + for (String query : queries) { + assertPlan(conn, query).serverGroupByLimit(null); } } @@ -2219,8 +2202,8 @@ public void testServerArrayElementProjection1() throws SQLException { Connection conn = DriverManager.getConnection(getUrl()); try { conn.createStatement().execute("CREATE TABLE t(a INTEGER PRIMARY KEY, arr INTEGER ARRAY)"); - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN SELECT arr[1] from t"); - assertTrue(QueryUtil.getExplainPlan(rs).contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertPlan(conn, "SELECT arr[1] from t").serverParsedProjections("ARRAY", + "ARRAY_ELEM(ARR, 1)"); } finally { conn.createStatement().execute("DROP TABLE IF EXISTS t"); conn.close(); @@ -2232,8 +2215,7 @@ public void testServerArrayElementProjection2() throws SQLException { Connection conn = DriverManager.getConnection(getUrl()); try { conn.createStatement().execute("CREATE TABLE t(a INTEGER PRIMARY KEY, arr INTEGER ARRAY)"); - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN SELECT arr, arr[1] from t"); - assertFalse(QueryUtil.getExplainPlan(rs).contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertPlan(conn, "SELECT arr, arr[1] from t").serverParsedProjectionsNone(); } finally { conn.createStatement().execute("DROP TABLE IF EXISTS t"); conn.close(); @@ -2246,9 +2228,8 @@ public void testServerArrayElementProjection3() throws SQLException { try { conn.createStatement() .execute("CREATE TABLE t(a INTEGER PRIMARY KEY, arr INTEGER ARRAY, arr2 VARCHAR ARRAY)"); - ResultSet rs = - conn.createStatement().executeQuery("EXPLAIN SELECT arr, arr[1], arr2[1] from t"); - assertTrue(QueryUtil.getExplainPlan(rs).contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertPlan(conn, "SELECT arr, arr[1], arr2[1] from t").serverParsedProjections("ARRAY", + "ARRAY_ELEM(ARR2, 1)"); } finally { conn.createStatement().execute("DROP TABLE IF EXISTS t"); conn.close(); @@ -2261,9 +2242,9 @@ public void testServerArrayElementProjection4() throws SQLException { try { conn.createStatement() .execute("CREATE TABLE t (p INTEGER PRIMARY KEY, arr1 INTEGER ARRAY, arr2 INTEGER ARRAY)"); - ResultSet rs = conn.createStatement().executeQuery( - "EXPLAIN SELECT arr1, arr1[1], ARRAY_APPEND(ARRAY_APPEND(arr1, arr2[2]), arr2[1]), p from t"); - assertTrue(QueryUtil.getExplainPlan(rs).contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertPlan(conn, + "SELECT arr1, arr1[1], ARRAY_APPEND(ARRAY_APPEND(arr1, arr2[2]), arr2[1]), p from t") + .serverParsedProjectionCount("ARRAY", 2); } finally { conn.createStatement().execute("DROP TABLE IF EXISTS t"); conn.close(); @@ -2321,9 +2302,9 @@ public void testServerArrayElementProjection5() throws SQLException { try { conn.createStatement() .execute("CREATE TABLE t (p INTEGER PRIMARY KEY, arr1 INTEGER ARRAY, arr2 INTEGER ARRAY)"); - ResultSet rs = conn.createStatement().executeQuery( - "EXPLAIN SELECT arr1, arr1[1], ARRAY_ELEM(ARRAY_APPEND(arr1, arr2[1]), 1), p, arr2[2] from t"); - assertTrue(QueryUtil.getExplainPlan(rs).contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertPlan(conn, + "SELECT arr1, arr1[1], ARRAY_ELEM(ARRAY_APPEND(arr1, arr2[1]), 1), p, arr2[2] from t") + .serverParsedProjectionCount("ARRAY", 2); } finally { conn.createStatement().execute("DROP TABLE IF EXISTS t"); conn.close(); @@ -2335,8 +2316,7 @@ public void testServerArrayElementProjectionWithArrayPrimaryKey() throws SQLExce Connection conn = DriverManager.getConnection(getUrl()); try { conn.createStatement().execute("CREATE TABLE t(arr INTEGER ARRAY PRIMARY KEY)"); - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN SELECT arr[1] from t"); - assertFalse(QueryUtil.getExplainPlan(rs).contains(" SERVER ARRAY ELEMENT PROJECTION")); + assertPlan(conn, "SELECT arr[1] from t").serverParsedProjectionsNone(); } finally { conn.createStatement().execute("DROP TABLE IF EXISTS t"); conn.close(); @@ -2569,27 +2549,22 @@ public void testFuncIndexUsage() throws SQLException { conn.createStatement() .execute("CREATE TABLE t3(j INTEGER PRIMARY KEY," + " col3 VARCHAR, col4 VARCHAR)"); conn.createStatement().execute("CREATE INDEX idx ON t1 (col1 || col2)"); - String query = "SELECT a.k from t1 a where a.col1 || a.col2 = 'foobar'"; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER IDX ['foobar']\n" - + " SERVER FILTER BY FIRST KEY ONLY", explainPlan); - query = "SELECT k,j from t3 b join t1 a ON k = j where a.col1 || a.col2 = 'foobar'"; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER T3\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER IDX ['foobar']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " DYNAMIC SERVER FILTER BY B.J IN (\"A.:K\")", explainPlan); - query = "SELECT a.k,b.k from t2 b join t1 a ON a.k = b.k where a.col1 || a.col2 = 'foobar'"; - rs = conn.createStatement().executeQuery("EXPLAIN " + query); - explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER T2\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" + " PARALLEL INNER-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER IDX ['foobar']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " DYNAMIC SERVER FILTER BY B.K IN (\"A.:K\")", explainPlan); + assertPlan(conn, "SELECT a.k from t1 a where a.col1 || a.col2 = 'foobar'") + .scanType("RANGE SCAN").table("IDX").keyRanges("['foobar']") + .serverFirstKeyOnlyProjection(true); + assertPlan(conn, "SELECT k,j from t3 b join t1 a ON k = j where a.col1 || a.col2 = 'foobar'") + .scanType("FULL SCAN").table("T3").serverFirstKeyOnlyProjection(true) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY B.J IN (\"A.:K\")").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table("IDX").keyRanges("['foobar']") + .serverFirstKeyOnlyProjection(true).end(); + assertPlan(conn, + "SELECT a.k,b.k from t2 b join t1 a ON a.k = b.k where a.col1 || a.col2 = 'foobar'") + .scanType("FULL SCAN").table("T2").serverFirstKeyOnlyProjection(true) + .dynamicServerFilter("DYNAMIC SERVER FILTER BY B.K IN (\"A.:K\")").subPlanCount(1) + .subPlan(0).abstractExplainPlan("PARALLEL INNER-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table("IDX").keyRanges("['foobar']") + .serverFirstKeyOnlyProjection(true).end(); } finally { conn.close(); } @@ -6301,10 +6276,7 @@ private static void verifyQueryPlanForSortMergeBug4508(Connection conn, String p + "JOIN " + peopleTable + " ds ON ds.PERSON_ID = l.LOCALID"; for (String q : new String[] { query1, query2 }) { - ResultSet rs = conn.createStatement().executeQuery("explain " + q); - String plan = QueryUtil.getExplainPlan(rs); - assertFalse("Tables should not require sort over their PKs:\n" + plan, - plan.contains("SERVER SORTED BY")); + assertPlan(conn, q).serverSortedBy(null).rhs().serverSortedBy(null); } } @@ -7179,12 +7151,9 @@ public void testReverseIndexRangeBugPhoenix6916() throws Exception { String query = "select id, ts from " + tableName + " where ts >= TIMESTAMP '2023-02-23 13:30:00' and ts < TIMESTAMP '2023-02-23 13:40:00'"; - ResultSet rs = stmt.executeQuery("EXPLAIN " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexName - + " [~1,677,159,600,000] - [~1,677,159,000,000]\n SERVER FILTER BY FIRST KEY ONLY", - explainPlan); + assertPlan(conn, query).scanType("RANGE SCAN").table(indexName) + .keyRanges("[~1,677,159,600,000] - [~1,677,159,000,000]").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedNone(); } } @@ -7196,27 +7165,23 @@ public void testReverseVarLengthRange6916() throws Exception { stmt.execute("create table " + tableName + " (k varchar primary key desc)"); - // Explain doesn't display open/closed ranges - String explainExpected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName - + " [~'aaa'] - [~'a']\n SERVER FILTER BY FIRST KEY ONLY"; - String openQry = "select * from " + tableName + " where k > 'a' and k<'aaa'"; Scan openScan = getOptimizedQueryPlan(openQry, Collections.emptyList()).getContext().getScan(); assertEquals("\\x9E\\x9E\\x9F\\x00", Bytes.toStringBinary(openScan.getStartRow())); assertEquals("\\x9E\\xFF", Bytes.toStringBinary(openScan.getStopRow())); - ResultSet rs = stmt.executeQuery("EXPLAIN " + openQry); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals(explainExpected, explainPlan); + assertPlan(conn, openQry).scanType("RANGE SCAN").table(tableName) + .keyRanges("[~'aaa'] - [~'a']").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); String closedQry = "select * from " + tableName + " where k >= 'a' and k <= 'aaa'"; Scan closedScan = getOptimizedQueryPlan(closedQry, Collections.emptyList()).getContext().getScan(); assertEquals("\\x9E\\x9E\\x9E\\xFF", Bytes.toStringBinary(closedScan.getStartRow())); assertEquals("\\x9F\\x00", Bytes.toStringBinary(closedScan.getStopRow())); - rs = stmt.executeQuery("EXPLAIN " + closedQry); - explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals(explainExpected, explainPlan); + assertPlan(conn, closedQry).scanType("RANGE SCAN").table(tableName) + .keyRanges("[~'aaa'] - [~'a']").serverFirstKeyOnlyProjection(true) + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); } } @@ -7232,14 +7197,11 @@ public void testUncoveredPhoenix6969() throws Exception { stmt.execute("create index ii on dd (k4, k1, k2, k3)"); String query = "select /*+ index(dd ii) */ k1, k2, k3, k4, v1, v2, v3, v4 from dd" + " where k4=1 and k2=1 order by k1 asc, v1 asc limit 1"; - ResultSet rs = stmt.executeQuery("EXPLAIN " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - // We are more interested in the query compiling than the exact result - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER II [1]\n" - + " SERVER MERGE [0.V1, 0.V2, 0.V3, 0.V4]\n" - + " SERVER FILTER BY FIRST KEY ONLY AND \"K2\" = 1\n" - + " SERVER TOP 1 ROW SORTED BY [\"K1\", \"V1\"]\n" + "CLIENT MERGE SORT\n" - + "CLIENT LIMIT 1", explainPlan); + assertPlan(conn, query).scanType("RANGE SCAN").table("II").keyRanges("[1]") + .serverMergeColumns("[0.V1, 0.V2, 0.V3, 0.V4]").serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY \"K2\" = 1").serverSortedBy("[\"K1\", \"V1\"]") + .serverRowLimit(1L).clientRowLimit(1).clientSortAlgo("CLIENT MERGE SORT") + .indexRule(OptimizerReasons.RULE_HINT).indexRejectedNone(); } } @@ -7255,16 +7217,16 @@ public void testUncoveredPhoenix6984() throws Exception { String query = "SELECT /*+ INDEX(D I), NO_INDEX_SERVER_MERGE */ * " + "FROM D " + "WHERE K2 = 'XXX' AND " + "V2 >= TIMESTAMP '2023-05-31 23:59:59.000' AND " + "V1 <= TIMESTAMP '2023-04-01 00:00:00.000' " + "ORDER BY V2 asc"; - ResultSet rs = stmt.executeQuery("EXPLAIN " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER D\n" - + " SERVER FILTER BY (V2 >= TIMESTAMP '2023-05-31 23:59:59.000'" - + " AND V1 <= TIMESTAMP '2023-04-01 00:00:00.000')\n" + " SERVER SORTED BY [D.V2]\n" - + "CLIENT MERGE SORT\n" + " SKIP-SCAN-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER I ['XXX']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " DYNAMIC SERVER FILTER BY (\"D.K1\", \"D.K2\", \"D.K3\", \"D.K4\")" - + " IN (($2.$4, $2.$5, $2.$6, $2.$7))", explainPlan); + assertPlan(conn, query).scanType("FULL SCAN").table("D") + .serverWhereFilter("SERVER FILTER BY (V2 >= TIMESTAMP '2023-05-31 23:59:59.000'" + + " AND V1 <= TIMESTAMP '2023-04-01 00:00:00.000')") + .serverSortedBy("[D.V2]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY (\"D.K1\", \"D.K2\", \"D.K3\", \"D.K4\")" + + " IN (($2.$4, $2.$5, $2.$6, $2.$7))") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("SKIP-SCAN-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table("I").keyRanges("['XXX']").serverFirstKeyOnlyProjection(true) + .end(); } } @@ -7284,17 +7246,17 @@ public void testUncoveredPhoenix6986() throws Exception { "SELECT /*+ INDEX(TAB_PHOENIX_6986 IDX_PHOENIX_6986) */ * " + "FROM TAB_PHOENIX_6986 " + "WHERE K2 = 'XXX' AND " + "V2 >= TIMESTAMP '2023-05-31 23:59:59.000' AND " + "V1 <= TIMESTAMP '2023-04-01 00:00:00.000' " + "ORDER BY V2 asc"; - ResultSet rs = stmt.executeQuery("EXPLAIN " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER TAB_PHOENIX_6986\n" - + " SERVER FILTER BY (V2 >= TIMESTAMP '2023-05-31 23:59:59.000'" - + " AND V1 <= TIMESTAMP '2023-04-01 00:00:00.000')\n" - + " SERVER SORTED BY [TAB_PHOENIX_6986.V2]\n" + "CLIENT MERGE SORT\n" - + " SKIP-SCAN-JOIN TABLE 0\n" - + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER IDX_PHOENIX_6986 ['XXX']\n" - + " SERVER FILTER BY FIRST KEY ONLY\n" - + " DYNAMIC SERVER FILTER BY (\"TAB_PHOENIX_6986.K1\", \"TAB_PHOENIX_6986.K2\", \"TAB_PHOENIX_6986.K3\", \"TAB_PHOENIX_6986.K4\")" - + " IN (($2.$4, $2.$5, $2.$6, $2.$7))", explainPlan); + assertPlan(conn, query).scanType("FULL SCAN").table("TAB_PHOENIX_6986") + .serverWhereFilter("SERVER FILTER BY (V2 >= TIMESTAMP '2023-05-31 23:59:59.000'" + + " AND V1 <= TIMESTAMP '2023-04-01 00:00:00.000')") + .serverSortedBy("[TAB_PHOENIX_6986.V2]").clientSortAlgo("CLIENT MERGE SORT") + .dynamicServerFilter("DYNAMIC SERVER FILTER BY (\"TAB_PHOENIX_6986.K1\"," + + " \"TAB_PHOENIX_6986.K2\", \"TAB_PHOENIX_6986.K3\", \"TAB_PHOENIX_6986.K4\")" + + " IN (($2.$4, $2.$5, $2.$6, $2.$7))") + .subPlanCount(1).subPlan(0) + .abstractExplainPlan("SKIP-SCAN-JOIN TABLE 0 /* HASH BUILD RIGHT */") + .scanType("RANGE SCAN").table("IDX_PHOENIX_6986").keyRanges("['XXX']") + .serverFirstKeyOnlyProjection(true).end(); } } @@ -7306,10 +7268,9 @@ public void testUncoveredPhoenix6961() throws Exception { "create table d (k integer primary key, v1 integer, v2 integer, v3 integer, v4 integer)"); stmt.execute("create index i on d(v2) include (v3)"); String query = "select /*+ index(d i) */ * from d where v2=1 and v3=1"; - ResultSet rs = stmt.executeQuery("EXPLAIN " + query); - String explainPlan = QueryUtil.getExplainPlan(rs); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER I [1]\n" - + " SERVER MERGE [0.V1, 0.V4]\n" + " SERVER FILTER BY \"V3\" = 1", explainPlan); + assertPlan(conn, query).scanType("RANGE SCAN").table("I").keyRanges("[1]") + .serverMergeColumns("[0.V1, 0.V4]").serverWhereFilter("SERVER FILTER BY \"V3\" = 1") + .indexRule(OptimizerReasons.RULE_HINT).indexRejectedNone(); } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java index 286ef8c29cc..d0e362ae9a5 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java @@ -21,6 +21,7 @@ import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.MIN_QUALIFIER; import static org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE; import static org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_NAME; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -35,15 +36,19 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Properties; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.LiteralExpression; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.parse.DeleteStatement; import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.parse.SQLParser; @@ -53,7 +58,6 @@ import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Ignore; @@ -502,11 +506,55 @@ public void testQueryOptimizerShouldSelectThePlanWithMoreNumberOfPKColumns() thr .execute("create index INDEX_TEST_TABLE_INDEX_D on INDEX_TEST_TABLE(A,D) include(B,C,E,F)"); conn1.createStatement() .execute("create index INDEX_TEST_TABLE_INDEX_F on INDEX_TEST_TABLE(A,F) include(B,C,D,E)"); - ResultSet rs = conn2.createStatement().executeQuery( - "explain select * from INDEX_TEST_TABLE where A in ('1','2','3','4','5') and F in ('1111','2222','3333')"); - assertEquals( - "CLIENT PARALLEL 1-WAY SKIP SCAN ON 15 KEYS OVER INDEX_TEST_TABLE_INDEX_F ['1','1111'] - ['5','3333']", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn2, + "select * from INDEX_TEST_TABLE where A in ('1','2','3','4','5') and F in ('1111','2222','3333')") + .scanType("SKIP SCAN ON 15 KEYS").table("INDEX_TEST_TABLE_INDEX_F") + .keyRanges("['1','1111'] - ['5','3333']") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedCount(1) + .indexRejected(0, "INDEX_TEST_TABLE_INDEX_D", OptimizerReasons.REASON_NO_PK_PREFIX_BOUND); + } + + @Test + public void testFunctionalIndexChosenRuleMatchesExpression() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute( + "create table fe_match (id varchar not null primary key, name varchar, val varchar)"); + conn.createStatement().execute("create index fe_match_upper_idx on fe_match (UPPER(name))"); + // The functional index's indexed expression matches the query's UPPER(NAME) path expression, + // so the chosen index carries a separate "matches " functional match disclosure with + // its selection rule. + assertPlan(conn, "select id from fe_match where UPPER(name) = 'ABC'") + .table("FE_MATCH_UPPER_IDX").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .functionalMatch("UPPER(NAME)"); + } + + @Test + public void testFunctionalIndexRejectedPathExpressionDoesNotMatch() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute( + "create table fe_nomatch (id varchar not null primary key, name varchar, val varchar, other varchar)"); + conn.createStatement().execute("create index fe_nomatch_upper_idx on fe_nomatch (UPPER(name))"); + conn.createStatement() + .execute("create index fe_nomatch_val_idx on fe_nomatch (val) include (other)"); + // The query never references UPPER(NAME), so the functional index matched no path expression + // and its generic "does not cover projection" rejection is swapped for the more specific + // "path expression does not match". + assertPlan(conn, "select val, other from fe_nomatch where val = 'x'") + .table("FE_NOMATCH_VAL_IDX").indexRejectedContains("FE_NOMATCH_UPPER_IDX", + OptimizerReasons.REASON_PATH_EXPRESSION_DOES_NOT_MATCH); + } + + @Test + public void testFunctionalIndexRejectedDoesNotCoverProjection() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute( + "create table fe_cover (id varchar not null primary key, name varchar, val varchar)"); + conn.createStatement().execute("create index fe_cover_upper_idx on fe_cover (UPPER(name))"); + // The functional index's expression DOES match UPPER(NAME) in the query, but VAL is not in the + // index, so the rejection stays "does not cover projection" rather than being swapped. + assertPlan(conn, "select UPPER(name), val from fe_cover where UPPER(name) = 'ABC'") + .table("FE_COVER").indexRejectedContains("FE_COVER_UPPER_IDX", + OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION); } @Test @@ -912,4 +960,44 @@ private static void assertQualifierRangesNotPresent(ResultSet rs) throws SQLExce assertNull(scan.getAttribute(MIN_QUALIFIER)); assertNull(scan.getAttribute(MAX_QUALIFIER)); } + + /** + * Test the deep copy contract of the {@link StatementContext} copy constructor and + * {@link StatementContext#adoptRewriteState}. + */ + @Test + public void testAdoptRewriteStateDoesNotBleedSourceMutations() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + + StatementContext source = new StatementContext(stmt); + source.addAppliedRewrite("X"); + source.recordAppliedIndexExpressionMatches("IDX", Collections.singletonList("A + B")); + Expression predicate = LiteralExpression.newConstant(1); + source.tagPredicate(predicate, "WHERE"); + + // Copy via the copy constructor, which shares copyRewriteStateFrom() with adoptRewriteState(). + StatementContext copy = new StatementContext(source); + + // The copy observed the breadcrumbs recorded before the copy. + assertTrue(copy.getAppliedRewrites().contains("X")); + assertEquals(Collections.singletonList("A + B"), copy.getAppliedIndexExpressionMatches("IDX")); + assertTrue(copy.getPredicateOrigins(predicate).contains("WHERE")); + + // Mutating the source after the copy must not leak into the destination. + source.addAppliedRewrite("foo"); + source.recordAppliedIndexExpressionMatches("IDX", Collections.singletonList("C + D")); + source.tagPredicate(predicate, "JOIN ON"); + + assertFalse(copy.getAppliedRewrites().contains("foo")); + assertFalse(copy.getAppliedIndexExpressionMatches("IDX").contains("C + D")); + assertFalse(copy.getPredicateOrigins(predicate).contains("JOIN ON")); + + // adoptRewriteState() onto a fresh destination has the same isolation guarantee. + StatementContext adopted = new StatementContext(stmt); + adopted.adoptRewriteState(source); + assertTrue(adopted.getAppliedRewrites().contains("foo")); + source.addAppliedRewrite("bar"); + assertFalse(adopted.getAppliedRewrites().contains("bar")); + } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/StatementHintsCompilationTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/StatementHintsCompilationTest.java index dd30f563c1d..7fb0f694ac8 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/StatementHintsCompilationTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/StatementHintsCompilationTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.compile; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -24,7 +25,6 @@ import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collections; import java.util.List; @@ -34,9 +34,9 @@ import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseConnectionlessQueryTest; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; @@ -99,26 +99,29 @@ public void testSelectForceRangeScanForEH() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); conn.createStatement().execute( "create table eh (organization_id char(15) not null,parent_id char(15) not null, created_date date not null, entity_history_id char(15) not null constraint pk primary key (organization_id, parent_id, created_date, entity_history_id))"); - ResultSet rs = conn.createStatement().executeQuery( - "explain select /*+ RANGE_SCAN */ ORGANIZATION_ID, PARENT_ID, CREATED_DATE, ENTITY_HISTORY_ID from eh where ORGANIZATION_ID='111111111111111' and SUBSTR(PARENT_ID, 1, 3) = 'foo' and CREATED_DATE >= TO_DATE ('2012-11-01 00:00:00') and CREATED_DATE < TO_DATE ('2012-11-30 00:00:00') order by ORGANIZATION_ID, PARENT_ID, CREATED_DATE DESC, ENTITY_HISTORY_ID limit 100"); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER EH ['111111111111111','foo ','2012-11-01 00:00:00.000'] - ['111111111111111','fop ','2012-11-30 00:00:00.000']\n" - + " SERVER FILTER BY FIRST KEY ONLY AND (CREATED_DATE >= DATE '2012-11-01 00:00:00.000' AND CREATED_DATE < DATE '2012-11-30 00:00:00.000')\n" - + " SERVER TOP 100 ROWS SORTED BY [ORGANIZATION_ID, PARENT_ID, CREATED_DATE DESC, ENTITY_HISTORY_ID]\n" - + "CLIENT MERGE SORT\nCLIENT LIMIT 100", - QueryUtil.getExplainPlan(rs)); + String query = + "select /*+ RANGE_SCAN */ ORGANIZATION_ID, PARENT_ID, CREATED_DATE, ENTITY_HISTORY_ID from eh where ORGANIZATION_ID='111111111111111' and SUBSTR(PARENT_ID, 1, 3) = 'foo' and CREATED_DATE >= TO_DATE ('2012-11-01 00:00:00') and CREATED_DATE < TO_DATE ('2012-11-30 00:00:00') order by ORGANIZATION_ID, PARENT_ID, CREATED_DATE DESC, ENTITY_HISTORY_ID limit 100"; + assertPlan(conn, query).scanType("RANGE SCAN").table("EH") + .keyRanges("['111111111111111','foo ','2012-11-01 00:00:00.000']" + + " - ['111111111111111','fop ','2012-11-30 00:00:00.000']") + .serverFirstKeyOnlyProjection(true) + .serverWhereFilter("SERVER FILTER BY (CREATED_DATE >= DATE" + + " '2012-11-01 00:00:00.000' AND CREATED_DATE < DATE '2012-11-30 00:00:00.000')") + .serverSortedBy("[ORGANIZATION_ID, PARENT_ID, CREATED_DATE DESC, ENTITY_HISTORY_ID]") + .serverRowLimit(100L).clientSortAlgo("CLIENT MERGE SORT").clientRowLimit(100) + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); } @Test public void testSerialHint() throws Exception { // test AggregatePlan String query = "SELECT /*+ SERIAL */ COUNT(*) FROM atable"; - assertTrue("Expected a SERIAL query", - compileStatement(query).getExplainPlan().getPlanSteps().get(0).contains("SERIAL")); + assertPlan(compileStatement(query).getExplainPlan().getPlanStepsAsAttributes()) + .iteratorType("SERIAL"); // test ScanPlan query = "SELECT /*+ SERIAL */ * FROM atable limit 10"; - assertTrue("Expected a SERIAL query", compileStatement(query, Collections.emptyList(), 10) - .getExplainPlan().getPlanSteps().get(0).contains("SERIAL")); + assertPlan(compileStatement(query, Collections.emptyList(), 10).getExplainPlan() + .getPlanStepsAsAttributes()).iteratorType("SERIAL"); } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java index 02af616cb44..33c8b6c44b7 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java @@ -17,20 +17,23 @@ */ package org.apache.phoenix.compile; +import static org.apache.phoenix.query.explain.ExplainPlanTestUtil.assertPlan; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.Calendar; +import java.util.List; import java.util.Properties; import java.util.TimeZone; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.query.explain.ExplainPlanTestUtil; import org.apache.phoenix.util.DateUtil; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.QueryUtil; import org.junit.Test; public class TenantSpecificViewIndexCompileTest extends BaseConnectionlessQueryTest { @@ -49,11 +52,10 @@ public void testOrderByOptimizedOut() throws Exception { conn.createStatement().execute("CREATE VIEW v(v2 VARCHAR) AS SELECT * FROM t WHERE k1 = 'a'"); conn.createStatement().execute("CREATE INDEX i1 ON v(v2) INCLUDE(v1)"); - ResultSet rs = - conn.createStatement().executeQuery("EXPLAIN SELECT v1,v2 FROM v WHERE v2 > 'a' ORDER BY v2"); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_T [-9223372036854775808,'me','a'] - [-9223372036854775808,'me',*]", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, "SELECT v1,v2 FROM v WHERE v2 > 'a' ORDER BY v2") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table("_IDX_T") + .keyRanges("[-9223372036854775808,'me','a'] - [-9223372036854775808,'me',*]") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS); } @Test @@ -69,35 +71,29 @@ public void testOrderByOptimizedOutWithoutPredicateInView() throws Exception { // Query without predicate ordered by full row key String sql = "SELECT * FROM v1 ORDER BY k1, k2, k3"; - String expectedExplainOutput = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789']"); assertOrderByHasBeenOptimizedOut(conn, sql); // Predicate with valid partial PK sql = "SELECT * FROM v1 WHERE k1 = 'xyz' ORDER BY k1, k2, k3"; - expectedExplainOutput = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz']"); assertOrderByHasBeenOptimizedOut(conn, sql); sql = "SELECT * FROM v1 WHERE k1 > 'xyz' ORDER BY k1, k2, k3"; - expectedExplainOutput = - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xy{'] - ['tenant123456789',*]"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xy{'] - ['tenant123456789',*]"); assertOrderByHasBeenOptimizedOut(conn, sql); String datePredicate = createStaticDate(); sql = "SELECT * FROM v1 WHERE k1 = 'xyz' AND k2 = '123456789012345' AND k3 < TO_DATE('" + datePredicate + "') ORDER BY k1, k2, k3"; - expectedExplainOutput = - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz','123456789012345',*] - ['tenant123456789','xyz','123456789012345','2015-01-01 08:00:00.000']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz','123456789012345',*] - " + + "['tenant123456789','xyz','123456789012345','2015-01-01 08:00:00.000']"); assertOrderByHasBeenOptimizedOut(conn, sql); // Predicate without valid partial PK sql = "SELECT * FROM v1 WHERE k2 < 'abcde1234567890' ORDER BY k1, k2, k3"; - expectedExplainOutput = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789']\n" - + " SERVER FILTER BY K2 < 'abcde1234567890'"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScanWithFilter(conn, sql, "['tenant123456789']", + "SERVER FILTER BY K2 < 'abcde1234567890'"); assertOrderByHasBeenOptimizedOut(conn, sql); } @@ -113,43 +109,36 @@ public void testOrderByOptimizedOutWithPredicateInView() throws Exception { // Query without predicate ordered by full row key String sql = "SELECT * FROM v1 ORDER BY k2, k3"; - String expectedExplainOutput = - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz']"); assertOrderByHasBeenOptimizedOut(conn, sql); // Query without predicate ordered by full row key, but without column view predicate sql = "SELECT * FROM v1 ORDER BY k2, k3"; - expectedExplainOutput = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz']"); assertOrderByHasBeenOptimizedOut(conn, sql); // Predicate with valid partial PK sql = "SELECT * FROM v1 WHERE k1 = 'xyz' ORDER BY k2, k3"; - expectedExplainOutput = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz']"); assertOrderByHasBeenOptimizedOut(conn, sql); sql = "SELECT * FROM v1 WHERE k2 < 'abcde1234567890' ORDER BY k2, k3"; - expectedExplainOutput = - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz',*] - ['tenant123456789','xyz','abcde1234567890']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, + "['tenant123456789','xyz',*] - ['tenant123456789','xyz','abcde1234567890']"); assertOrderByHasBeenOptimizedOut(conn, sql); // Predicate with full PK String datePredicate = createStaticDate(); sql = "SELECT * FROM v1 WHERE k2 = '123456789012345' AND k3 < TO_DATE('" + datePredicate + "') ORDER BY k2, k3"; - expectedExplainOutput = - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz','123456789012345',*] - ['tenant123456789','xyz','123456789012345','2015-01-01 08:00:00.000']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz','123456789012345',*] - " + + "['tenant123456789','xyz','123456789012345','2015-01-01 08:00:00.000']"); assertOrderByHasBeenOptimizedOut(conn, sql); // Predicate with valid partial PK sql = "SELECT * FROM v1 WHERE k3 < TO_DATE('" + datePredicate + "') ORDER BY k2, k3"; - expectedExplainOutput = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz']\n" - + " SERVER FILTER BY K3 < DATE '" + datePredicate + "'"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScanWithFilter(conn, sql, "['tenant123456789','xyz']", + "SERVER FILTER BY K3 < DATE '" + datePredicate + "'"); assertOrderByHasBeenOptimizedOut(conn, sql); } @@ -166,30 +155,27 @@ public void testOrderByOptimizedOutWithMultiplePredicatesInView() throws Excepti // Query without predicate ordered by full row key String sql = "SELECT * FROM v1 ORDER BY k3 DESC"; - String expectedExplainOutput = - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz','abcde']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz','abcde']"); assertOrderByHasBeenOptimizedOut(conn, sql); // Query without predicate ordered by full row key, but without column view predicate sql = "SELECT * FROM v1 ORDER BY k3 DESC"; - expectedExplainOutput = - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz','abcde']"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz','abcde']"); assertOrderByHasBeenOptimizedOut(conn, sql); // Query with predicate ordered by full row key sql = "SELECT * FROM v1 WHERE k3 <= TO_DATE('" + createStaticDate() + "') ORDER BY k3 DESC"; - expectedExplainOutput = - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz','abcde',~'2015-01-01 08:00:00.000'] - ['tenant123456789','xyz','abcde',*]"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertRangeScan(conn, sql, "['tenant123456789','xyz','abcde',~'2015-01-01 08:00:00.000'] - " + + "['tenant123456789','xyz','abcde',*]"); assertOrderByHasBeenOptimizedOut(conn, sql); // Query with predicate ordered by full row key with date in reverse order sql = "SELECT * FROM v1 WHERE k3 <= TO_DATE('" + createStaticDate() + "') ORDER BY k3"; - expectedExplainOutput = - "CLIENT PARALLEL 1-WAY REVERSE RANGE SCAN OVER T ['tenant123456789','xyz','abcde',~'2015-01-01 08:00:00.000'] - ['tenant123456789','xyz','abcde',*]"; - assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput); + assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table("T") + .clientSortedBy("REVERSE") + .keyRanges("['tenant123456789','xyz','abcde',~'2015-01-01 08:00:00.000'] - " + + "['tenant123456789','xyz','abcde',*]") + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); assertOrderByHasBeenOptimizedOut(conn, sql); } @@ -208,25 +194,29 @@ public void testViewConstantsOptimizedOut() throws Exception { conn.createStatement().execute("CREATE VIEW v(v2 VARCHAR) AS SELECT * FROM t WHERE k2 = 'a'"); conn.createStatement().execute("CREATE INDEX i1 ON v(v2)"); - ResultSet rs = conn.createStatement() - .executeQuery("EXPLAIN SELECT v2 FROM v WHERE v2 > 'a' and k2 = 'a' ORDER BY v2,k2"); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_T [-9223372036854775808,'me','a'] - [-9223372036854775808,'me',*]\n" - + " SERVER FILTER BY FIRST KEY ONLY", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, "SELECT v2 FROM v WHERE v2 > 'a' and k2 = 'a' ORDER BY v2,k2") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table("_IDX_T") + .keyRanges("[-9223372036854775808,'me','a'] - [-9223372036854775808,'me',*]") + .serverFirstKeyOnlyProjection(true).indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS); // Won't use index b/c v1 is not in index, but should optimize out k2 still from the order by // K2 will still be referenced in the filter, as these are automatically tacked on to the where // clause. - rs = conn.createStatement().executeQuery("EXPLAIN SELECT v1 FROM v WHERE v2 > 'a' ORDER BY k2"); - assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['me']\n" - + " SERVER FILTER BY (V2 > 'a' AND K2 = 'a')", QueryUtil.getExplainPlan(rs)); + // The index i1 is rejected because it does not cover v1, leaving the data table as the only + // surviving candidate. + assertPlan(conn, "SELECT v1 FROM v WHERE v2 > 'a' ORDER BY k2").iteratorType("PARALLEL 1-WAY") + .scanType("RANGE SCAN").table("T").keyRanges("['me']") + .serverWhereFilter("SERVER FILTER BY (V2 > 'a' AND K2 = 'a')") + .indexRule(OptimizerReasons.RULE_ONLY_CANDIDATE).indexRejectedCount(1); // If we match K2 against a constant not equal to it's view constant, we should get a degenerate - // plan - rs = conn.createStatement() - .executeQuery("EXPLAIN SELECT v1 FROM v WHERE v2 > 'a' and k2='b' ORDER BY k2"); - assertEquals("DEGENERATE SCAN OVER V", QueryUtil.getExplainPlan(rs)); + // plan. The DEGENERATE SCAN ExplainPlan does not populate structured attributes, so this + // single-literal check on the plan-steps text is retained. + List degenerateSteps = ExplainPlanTestUtil.getPlanSteps(conn, + "SELECT v1 FROM v WHERE v2 > 'a' and k2='b' ORDER BY k2"); + assertEquals(1, degenerateSteps.size()); + assertTrue("expected DEGENERATE SCAN OVER V, got " + degenerateSteps, + degenerateSteps.get(0).contains("DEGENERATE SCAN OVER V")); } @Test @@ -246,16 +236,16 @@ public void testViewConstantsOptimizedOutOnReadOnlyView() throws Exception { // Confirm that a read-only view on an updatable view still optimizes out the read-only parts of // the updatable view - ResultSet rs = conn.createStatement() - .executeQuery("EXPLAIN SELECT v2 FROM v2 WHERE v3 > 'a' and k2 = 'a' ORDER BY v3,k2"); - assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_T [-9223372036854775808,'me','a'] - [-9223372036854775808,'me',*]", - QueryUtil.getExplainPlan(rs)); + assertPlan(conn, "SELECT v2 FROM v2 WHERE v3 > 'a' and k2 = 'a' ORDER BY v3,k2") + .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table("_IDX_T") + .keyRanges("[-9223372036854775808,'me','a'] - [-9223372036854775808,'me',*]") + .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS); } // ----------------------------------------------------------------- // Private Helper Methods // ----------------------------------------------------------------- + private Connection createTenantSpecificConnection() throws SQLException { Connection conn; Properties props = new Properties(); @@ -265,10 +255,16 @@ private Connection createTenantSpecificConnection() throws SQLException { return conn; } - private void assertExplainPlanIsCorrect(Connection conn, String sql, String expectedExplainOutput) - throws SQLException { - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + sql); - assertEquals(expectedExplainOutput, QueryUtil.getExplainPlan(rs)); + private void assertRangeScan(Connection conn, String sql, String keyRanges) throws SQLException { + assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table("T") + .keyRanges(keyRanges).indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); + } + + private void assertRangeScanWithFilter(Connection conn, String sql, String keyRanges, + String serverWhereFilter) throws SQLException { + assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").table("T") + .keyRanges(keyRanges).serverWhereFilter(serverWhereFilter) + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedNone(); } private void assertOrderByHasBeenOptimizedOut(Connection conn, String sql) throws SQLException { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeExpectedRegenerator.java b/phoenix-core/src/test/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeExpectedRegenerator.java new file mode 100644 index 00000000000..e6f424939c9 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/end2end/tpcds/TPCDSLikeExpectedRegenerator.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end.tpcds; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Developer only tool that prints the embedded {@code

} rewrite breadcrumb on the chosen + * plan's context. The breadcrumb is rendered as a {@code REWRITE INDEX EXPRESSION ...} + * top-of-plan line in plain EXPLAIN text and as a single entry in the structured {@code rewrites} + * attribute. + */ + @Test + public void testIndexExpressionRewriteEmittedForChosenFunctionalIndex() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String base = generateUniqueName(); + String idx = generateUniqueName(); + stmt.execute("CREATE TABLE " + base + " (pk VARCHAR PRIMARY KEY, payload BSON)"); + stmt.execute("CREATE INDEX " + idx + " ON " + base + + " (BSON_VALUE(payload, 'k', 'VARCHAR')) INCLUDE (payload)"); + String query = "SELECT BSON_VALUE(payload, 'k', 'VARCHAR') FROM " + base + + " WHERE BSON_VALUE(payload, 'k', 'VARCHAR') = 'x'"; + + // The functional index is chosen and the separate functional match disclosure names the + // matched expression. + ExplainPlanTestUtil.assertPlan(conn, query).indexName(idx) + .functionalMatch("BSON_VALUE(PAYLOAD,'k','VARCHAR')") + // Exactly one breadcrumb (one applied substitution; no eager per-PK-column emissions). + .rewriteCount(1).rewrite(0, + "INDEX EXPRESSION BSON_VALUE(PAYLOAD,'k','VARCHAR') AS \":BSON_VALUE(PAYLOAD,'k','VARCHAR')\""); + + // Plain EXPLAIN renders the breadcrumb as a REWRITE top-of-plan line. + List rows = explainViaJdbc(conn, query); + assertEquals("REWRITE INDEX EXPRESSION BSON_VALUE(PAYLOAD,'k','VARCHAR') AS" + + " \":BSON_VALUE(PAYLOAD,'k','VARCHAR')\"", rows.get(0)); + assertTrue("operator should follow the REWRITE line: " + rows.get(1), + rows.get(1).startsWith("CLIENT")); + } + } + + /** + * A query that references the indexed path expression more than once must still emit the + * {@code INDEX EXPRESSION AS } breadcrumb exactly once. Guards against duplicates. + */ + @Test + public void testIndexExpressionRewriteEmittedOnceForRepeatedQueryReference() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String base = generateUniqueName(); + String idx = generateUniqueName(); + stmt.execute("CREATE TABLE " + base + " (pk VARCHAR PRIMARY KEY, payload BSON)"); + stmt.execute("CREATE INDEX " + idx + " ON " + base + + " (BSON_VALUE(payload, 'k', 'VARCHAR')) INCLUDE (payload)"); + // The same path expression appears in the WHERE, the projection, and the ORDER BY. + String query = "SELECT BSON_VALUE(payload, 'k', 'VARCHAR') FROM " + base + + " WHERE BSON_VALUE(payload, 'k', 'VARCHAR') >= 'a'" + + " ORDER BY BSON_VALUE(payload, 'k', 'VARCHAR')"; + ExplainPlanTestUtil.assertPlan(conn, query).indexName(idx).rewriteCount(1).rewrite(0, + "INDEX EXPRESSION BSON_VALUE(PAYLOAD,'k','VARCHAR') AS \":BSON_VALUE(PAYLOAD,'k','VARCHAR')\""); + } + } + + /** + * A plain global index chosen for a query must not emit an + * {@code INDEX EXPRESSION AS } breadcrumb for ordinary indexed columns. + */ + @Test + public void testIndexExpressionRewriteOmittedForNonFunctionalIndex() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String base = generateUniqueName(); + String idx = generateUniqueName(); + stmt.execute("CREATE TABLE " + base + " (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + stmt.execute("CREATE INDEX " + idx + " ON " + base + " (v1) INCLUDE (v2)"); + String query = "SELECT v1, v2 FROM " + base + " WHERE v1 = 'x'"; + ExplainPlanTestUtil.assertPlan(conn, query).indexName(idx).rewritesNone(); + assertNoPlanLineContains(conn, query, "INDEX EXPRESSION"); + } + } + + /** + * A functional index that is considered but rejected must not emit an + * {@code INDEX EXPRESSION AS } breadcrumb. + */ + @Test + public void testIndexExpressionRewriteOmittedWhenFunctionalIndexNotChosen() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String base = generateUniqueName(); + String idx = generateUniqueName(); + stmt.execute("CREATE TABLE " + base + " (pk VARCHAR PRIMARY KEY, payload BSON)"); + stmt.execute("CREATE INDEX " + idx + " ON " + base + + " (BSON_VALUE(payload, 'k', 'VARCHAR')) INCLUDE (payload)"); + // The functional index is over the 'k' path. The query reads the 'other' path and is on + // the data-table primary key, so the optimizer never substitutes the indexed expression. + String query = + "SELECT BSON_VALUE(payload, 'other', 'VARCHAR') FROM " + base + " WHERE pk = 'p1'"; + ExplainPlanTestUtil.assertPlan(conn, query).indexName(base).rewritesNone(); + assertNoPlanLineContains(conn, query, "INDEX EXPRESSION"); + } + } + + @Test + public void testVerboseProjectLine() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt + .execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR, b VARCHAR, c VARCHAR)"); + String query = "SELECT a, b FROM " + t; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverProject("A", "B"); + List verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue("expected VERBOSE plan to contain the PROJECT line but was " + verboseSteps, + verboseSteps.contains(" PROJECT A, B")); + // Plain EXPLAIN carries no PROJECT line and no serverProject attribute. + ExplainPlanTestUtil.assertPlan(conn, query).serverProjectNone(); + assertNoPlanLineContains(conn, query, "PROJECT "); + } + } + + @Test + public void testVerboseServerFilterWhereFanout() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt + .execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR, b VARCHAR, c VARCHAR)"); + String query = "SELECT a FROM " + t + " WHERE b = 'x' AND c = 'y'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverFilterCount(2) + .serverFilterOrigin(0, "WHERE").serverFilterPathTest(0, null).serverFilterOrigin(1, "WHERE") + .serverFilterPathTest(1, null); + // Plain EXPLAIN keeps the combined single serverWhereFilter line, no per-predicate breakdown. + ExplainPlanTestUtil.assertPlan(conn, query).serverFiltersNone(); + } + } + + @Test + public void testVerboseServerFilterSinglePredicate() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR, b VARCHAR)"); + String query = "SELECT a FROM " + t + " WHERE b = 'x'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverFilterCount(1) + .serverFilterOrigin(0, "WHERE").serverFilterPathTest(0, null); + } + } + + @Test + public void testVerboseServerFilterJsonExistsSubtag() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (pk VARCHAR PRIMARY KEY, jsoncol JSON)"); + String query = "SELECT pk FROM " + t + " WHERE JSON_EXISTS(jsoncol, '$.info.address.town')"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverFilterCount(1) + .serverFilterOrigin(0, "WHERE").serverFilterPathTest(0, "JSON EXISTS"); + } + } + + @Test + public void testVerboseServerFilterBsonConditionSubtag() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (pk VARCHAR PRIMARY KEY, payload BSON)"); + String query = "SELECT pk FROM " + t + + " WHERE BSON_CONDITION_EXPRESSION(payload, '{\"$EXPR\": \"field_exists(Id)\"}')"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverFilterCount(1) + .serverFilterOrigin(0, "WHERE").serverFilterPathTest(0, "BSON CONDITION"); + } + } + + @Test + public void testVerboseClientFilterFanout() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + String query = "SELECT a_string FROM (SELECT a_string, a_integer FROM atable LIMIT 5)" + + " WHERE a_integer > 2"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).clientFilterCount(1) + .clientFilter(0, "A_INTEGER > 2").clientFilterOrigin(0, "WHERE") + .clientFilterPathTest(0, null); + List verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue("expected VERBOSE plan to contain a CLIENT FILTER BY line but was " + verboseSteps, + verboseSteps.stream().anyMatch(s -> s.startsWith("CLIENT FILTER BY A_INTEGER > 2"))); + // Plain EXPLAIN keeps the combined clientFilterBy string and no structured breakdown. + ExplainPlanTestUtil.assertPlan(conn, query).clientFiltersNone() + .clientFilterBy("A_INTEGER > 2"); + } + } + + @Test + public void testVerboseClientFilterHavingFanout() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + String query = + "SELECT count(1) FROM atable GROUP BY a_string, b_string HAVING max(a_string) = 'a'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).clientFilterCount(1) + .clientFilter(0, "MAX(A_STRING) = 'a'").clientFilterOrigin(0, "HAVING") + .clientFilterPathTest(0, null); + // Plain EXPLAIN keeps the combined string and no structured breakdown. + ExplainPlanTestUtil.assertPlan(conn, query).clientFiltersNone() + .clientFilterBy("MAX(A_STRING) = 'a'"); + } + } + + /** + * When a top-level AND is only partially tagged, renderVerboseFilters collapses to a single + * combined line. That line must still union the origins recorded on the tagged conjunct(s) rather + * than reading only the parent expression. Normal WHERE/HAVING compilation always tags every + * conjunct, so this partial-tag state is reachable only when identity is lost during expression + * rewriting. + */ + @Test + public void testVerboseCombinedFilterUnionsConjunctOrigins() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + QueryPlan plan = conn.prepareStatement("SELECT * FROM atable") + .unwrap(PhoenixPreparedStatement.class).optimizeQuery(); + StatementContext context = plan.getContext(); + Expression tagged = LiteralExpression.newConstant(true); + Expression untagged = LiteralExpression.newConstant(false); + Expression and = new AndExpression(Arrays.asList(tagged, untagged)); + context.tagPredicate(tagged, "WHERE"); + List planSteps = new ArrayList<>(); + List filters = ExplainTable.renderVerboseFilters(context, and, and.toString(), + " SERVER FILTER BY", planSteps); + assertEquals("partial tagging must collapse to one combined line", 1, filters.size()); + assertEquals("combined line must union origins from tagged conjuncts", + Collections.singletonList("WHERE"), filters.get(0).getOrigin()); + assertEquals(1, planSteps.size()); + assertTrue("combined line should carry the unioned origin comment but was " + planSteps, + planSteps.get(0).endsWith("-- WHERE")); + } + } + + @Test + public void testVerboseIgnoredHintNoIndexNoIndexes() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR, b VARCHAR)"); + String query = "SELECT /*+ NO_INDEX */ a FROM " + t + " WHERE b = 'x'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).ignoredHint("NO_INDEX", + "no indexes on table"); + List verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue( + "expected VERBOSE plan to disclose the ignored NO_INDEX hint but was " + verboseSteps, + verboseSteps.contains(" /*- NO_INDEX -- no indexes on table */")); + // Plain EXPLAIN does not disclose ignored hints. + ExplainPlanTestUtil.assertPlan(conn, query).ignoredHintsNone(); + assertNoPlanLineContains(conn, query, "/*- NO_INDEX"); + } + } + + @Test + public void testVerboseIgnoredHintIndexNoMatch() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + String idx = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + stmt.execute("CREATE INDEX " + idx + " ON " + t + " (v1) INCLUDE (v2)"); + // Hint references an index name that does not exist on the table. + String query = + "SELECT /*+ INDEX(" + t + " NONEXISTENT_IDX) */ k, v2 FROM " + t + " WHERE v1 = 'x'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).ignoredHint("INDEX", + "no matching index applicable"); + } + } + + @Test + public void testVerboseIgnoredHintSortMergeNoJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR)"); + String query = "SELECT /*+ USE_SORT_MERGE_JOIN */ a FROM " + t; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).ignoredHint("USE_SORT_MERGE_JOIN", + "no join in query"); + List verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue( + "expected VERBOSE plan to disclose the ignored USE_SORT_MERGE_JOIN hint but was " + + verboseSteps, + verboseSteps.contains(" /*- USE_SORT_MERGE_JOIN -- no join in query */")); + } + } + + @Test + public void testRewriteReverseScanSubstitution() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil.assertPlan(conn, "SELECT inst,\"DATE\" FROM ptsdb2 WHERE inst = 'na1'" + + " ORDER BY inst DESC, \"DATE\" DESC").rewriteContains("REVERSE SCAN SUBSTITUTION"); + } + } + + @Test + public void testRewriteInSubqueryAsSemiJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil + .assertPlan(conn, + "SELECT a_string FROM atable" + + " WHERE organization_id IN (SELECT organization_id FROM atable WHERE a_integer = 1)") + .rewriteContains("IN SUBQUERY AS SEMI JOIN"); + } + } + + @Test + public void testRewriteNotInSubqueryAsAntiJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil.assertPlan(conn, "SELECT a_string FROM atable" + + " WHERE organization_id NOT IN (SELECT organization_id FROM atable WHERE a_integer = 1)") + .rewriteContains("NOT IN SUBQUERY AS ANTI JOIN"); + } + } + + @Test + public void testRewriteExistsSubqueryAsSemiJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil.assertPlan(conn, + "SELECT a_string FROM atable a WHERE EXISTS" + + " (SELECT 1 FROM atable b WHERE b.organization_id = a.organization_id" + + " AND b.a_integer = 1)") + .rewriteContains("EXISTS SUBQUERY AS SEMI JOIN"); + } + } + + @Test + public void testRewriteNotExistsSubqueryAsAntiJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil.assertPlan(conn, + "SELECT a_string FROM atable a WHERE NOT EXISTS" + + " (SELECT 1 FROM atable b WHERE b.organization_id = a.organization_id" + + " AND b.a_integer = 1)") + .rewriteContains("NOT EXISTS SUBQUERY AS ANTI JOIN"); + } + } + + @Test + public void testRewriteScalarSubqueryAsInnerJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil + .assertPlan(conn, + "SELECT a_string FROM atable a WHERE a_integer =" + + " (SELECT max(a_integer) FROM atable b WHERE b.organization_id = a.organization_id)") + .rewriteContains("SCALAR SUBQUERY AS INNER JOIN"); + } + } + + @Test + public void testRewriteHavingPredicateAsWhere() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil + .assertPlan(conn, "SELECT count(1) FROM atable GROUP BY a_string HAVING a_string = 'a'") + .rewriteContains("HAVING PREDICATE AS WHERE"); + } + } + + @Test + public void testRewriteDerivedTableFlattened() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil + .assertPlan(conn, "SELECT a_string FROM (SELECT a_string FROM atable) WHERE a_string = 'a'") + .rewriteContains("DERIVED TABLE FLATTENED 1"); + } + } + + @Test + public void testRewriteUnionOrderByMerge() throws Exception { + // The UNION ORDER BY merge optimization only fires when a union is the inner select of an + // outer query whose GROUP BY / ORDER BY is order-preserving over the union output. + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + props.setProperty(DATE_FORMAT_ATTRIB, "yyyy-MM-dd"); + props.setProperty(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.toString(false)); + try (Connection conn = DriverManager.getConnection(getUrl(), props); + java.sql.Statement stmt = conn.createStatement()) { + String t1 = generateUniqueName(); + String t2 = generateUniqueName(); + stmt.execute("CREATE TABLE " + t1 + " (fuid UNSIGNED_LONG NOT NULL," + + " fstatsdate UNSIGNED_LONG NOT NULL, faid_1 UNSIGNED_LONG NOT NULL," + + " clk_pv_1 UNSIGNED_LONG CONSTRAINT pk PRIMARY KEY (fuid, fstatsdate, faid_1))"); + stmt.execute("CREATE TABLE " + t2 + " (fuid UNSIGNED_LONG NOT NULL," + + " fstatsdate UNSIGNED_LONG NOT NULL, faid_2 UNSIGNED_LONG NOT NULL," + + " clk_pv_2 UNSIGNED_LONG CONSTRAINT pk PRIMARY KEY (fuid, fstatsdate, faid_2))"); + String unionSql = "(SELECT fuid AS advertiser_id, faid_1 AS adgroup_id," + + " fstatsdate AS date, SUM(clk_pv_1) AS clicks FROM " + t1 + + " GROUP BY fuid, faid_1, fstatsdate UNION ALL" + + " SELECT fuid AS advertiser_id, faid_2 AS adgroup_id," + + " fstatsdate AS date, SUM(clk_pv_2) AS clicks FROM " + t2 + + " GROUP BY fuid, faid_2, fstatsdate)"; + String sql = "SELECT advertiser_id, adgroup_id, date, SUM(clicks) AS clicks FROM " + unionSql + + " GROUP BY advertiser_id, adgroup_id, date ORDER BY advertiser_id, adgroup_id, date"; + ExplainPlanTestUtil.assertPlan(conn, sql).rewriteContains("UNION ORDER BY MERGE"); + } + } + + @Test + public void testRewriteStarJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil + .assertPlan(conn, + "SELECT a.a_string, b.a_string, c.a_string FROM atable a" + + " JOIN atable b ON a.organization_id = b.organization_id" + + " JOIN atable c ON a.organization_id = c.organization_id" + + " WHERE a.organization_id = '00D000000000001'") + .rewriteContains("STAR JOIN ON 2 RIGHT LEGS"); + } + } + + @Test + public void testRewriteRightJoinAsLeftJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil + .assertPlan(conn, + "SELECT a.a_string, b.a_string FROM atable a" + + " RIGHT JOIN atable b ON a.organization_id = b.organization_id" + + " WHERE b.organization_id = '00D000000000001'") + .rewriteContains("RIGHT JOIN AS LEFT JOIN"); + } + } + + @Test + public void testDisclosureTenantAndView() throws Exception { + Properties tenantProps = PropertiesUtil.deepCopy(TEST_PROPERTIES); + tenantProps.setProperty(DATE_FORMAT_ATTRIB, "yyyy-MM-dd"); + tenantProps.setProperty(TENANT_ID_ATTRIB, TENANT_ID); + try (Connection conn = DriverManager.getConnection(getUrl(), tenantProps)) { + ExplainPlanTestUtil.assertPlan(conn, "SELECT * FROM " + MT_VIEW + " LIMIT 1") + .tenant(TENANT_ID).view(MT_VIEW, MT_BASE); + } + } + + @Test + public void testDisclosureNoneOnPlainScan() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + ExplainPlanTestUtil.assertPlan(conn, "SELECT * FROM atable").tenantNone().viewNone() + .cdcScopesNone().txnProviderNone().rewritesNone(); + } + } + + /** + * End-to-end check that the disclosure block is prepended to the real JDBC {@code EXPLAIN} result + * set (the {@link org.apache.phoenix.jdbc.PhoenixStatement} path that invokes + * {@link ExplainTable#renderTopOfPlanText}), with {@code TENANT} then {@code VIEW} ahead of the + * first operator line. + */ + @Test + public void testTopOfPlanTextTenantAndView() throws Exception { + Properties tenantProps = PropertiesUtil.deepCopy(TEST_PROPERTIES); + tenantProps.setProperty(DATE_FORMAT_ATTRIB, "yyyy-MM-dd"); + tenantProps.setProperty(TENANT_ID_ATTRIB, TENANT_ID); + try (Connection conn = DriverManager.getConnection(getUrl(), tenantProps)) { + List rows = explainViaJdbc(conn, "SELECT * FROM " + MT_VIEW + " LIMIT 1"); + assertEquals("TENANT '" + TENANT_ID + "'", rows.get(0)); + assertEquals("VIEW " + MT_VIEW + " OVER " + MT_BASE, rows.get(1)); + assertTrue("operator should follow the disclosure block: " + rows.get(2), + rows.get(2).startsWith("CLIENT")); + } + } + + /** + * End-to-end check that a {@code REWRITE} breadcrumb is prepended ahead of the first operator. + */ + @Test + public void testTopOfPlanTextRewriteAtTop() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + List rows = + explainViaJdbc(conn, "SELECT a.a_string, b.a_string FROM atable a RIGHT JOIN atable b" + + " ON a.organization_id = b.organization_id WHERE b.organization_id = '00D000000000001'"); + assertEquals("REWRITE RIGHT JOIN AS LEFT JOIN", rows.get(0)); + assertTrue("operator should follow the REWRITE line: " + rows.get(1), + rows.get(1).startsWith("CLIENT")); + } + } + + /** A view with no resolvable base table renders {@code VIEW } without {@code OVER}. */ + @Test + public void testRenderTopOfPlanTextViewWithoutBase() { + ExplainPlanAttributes attrs = new ExplainPlanAttributesBuilder().setViewName("V").build(); + List steps = new java.util.ArrayList<>(Arrays.asList("CLIENT FOO")); + ExplainTable.renderTopOfPlanText(steps, attrs); + assertEquals(Arrays.asList("VIEW V", "CLIENT FOO"), steps); + } + + /** {@link ExplainTable#renderTopOfPlanText} leaves the steps untouched. */ + @Test + public void testRenderTopOfPlanTextNoopWhenNoDisclosure() { + ExplainPlanAttributes attrs = new ExplainPlanAttributesBuilder().build(); + List steps = new java.util.ArrayList<>( + Arrays.asList("CLIENT PARALLEL 1-WAY FULL SCAN OVER T", " SERVER FILTER BY X")); + ExplainTable.renderTopOfPlanText(steps, attrs); + assertEquals(Arrays.asList("CLIENT PARALLEL 1-WAY FULL SCAN OVER T", " SERVER FILTER BY X"), + steps); + } + + private static List explainViaJdbc(Connection conn, String query) throws SQLException { + List rows = new java.util.ArrayList<>(); + try (java.sql.Statement stmt = conn.createStatement(); + java.sql.ResultSet rs = stmt.executeQuery("EXPLAIN " + query)) { + while (rs.next()) { + rows.add(rs.getString(1)); + } + } + return rows; + } + + @Test + public void testTextNormalizerCollapsesWayCount() { + assertEquals(Collections.singletonList("CLIENT PARALLEL -WAY FULL SCAN OVER ATABLE"), + new ExplainTextNormalizer() + .normalize(Arrays.asList("CLIENT PARALLEL 400-WAY FULL SCAN OVER ATABLE"))); + } + + @Test + public void testTextNormalizerCollapsesChunkCount() { + assertEquals( + Collections.singletonList("CLIENT -CHUNK PARALLEL -WAY FULL SCAN OVER ATABLE"), + new ExplainTextNormalizer() + .normalize(Arrays.asList("CLIENT 5-CHUNK PARALLEL 16-WAY FULL SCAN OVER ATABLE"))); + } + + @Test + public void testTextNormalizerStripsRowsBytes() { + assertEquals( + Collections.singletonList("CLIENT -CHUNK PARALLEL -WAY FULL SCAN OVER ATABLE"), + new ExplainTextNormalizer().normalize( + Arrays.asList("CLIENT 1-CHUNK 100 ROWS 2048 BYTES PARALLEL 1-WAY FULL SCAN OVER ATABLE"))); + } + + @Test + public void testTextNormalizerDropsRegionLocationsLine() { + assertEquals( + Arrays.asList("CLIENT PARALLEL -WAY FULL SCAN OVER ATABLE", + " SERVER FILTER BY FIRST KEY ONLY"), + new ExplainTextNormalizer().normalize(Arrays.asList( + "CLIENT PARALLEL 1-WAY FULL SCAN OVER ATABLE", " SERVER FILTER BY FIRST KEY ONLY", + " (region locations = [{startKey=\\x00, endKey=, server=foo,1234}])"))); + } + + @Test + public void testTextNormalizerRenumbersTempAliasesByFirstAppearance() { + // Two distinct aliases ($7, $9) with $7 appearing first → $1, $9 → $2. + assertEquals( + Collections.singletonList(" DYNAMIC SERVER FILTER BY ATABLE.ORGANIZATION_ID IN ($1.$2)"), + new ExplainTextNormalizer().normalize(Collections + .singletonList(" DYNAMIC SERVER FILTER BY ATABLE.ORGANIZATION_ID IN ($7.$9)"))); + } + + @Test + public void testTextNormalizerSharesAliasStateAcrossLines() { + // The same alias ($5) appearing in two different lines gets the same renumbered token. A + // distinct alias ($8) on the second line gets the next number. + List in = Arrays.asList("CLIENT PARALLEL 1-WAY FULL SCAN OVER ($5)", + " DYNAMIC SERVER FILTER BY T.X IN ($5.$8)"); + assertEquals(Arrays.asList("CLIENT PARALLEL -WAY FULL SCAN OVER ($1)", + " DYNAMIC SERVER FILTER BY T.X IN ($1.$2)"), new ExplainTextNormalizer().normalize(in)); + } + + @Test + public void testTextNormalizerLeavesNonAliasDollarTokensAlone() { + // The "$" form below is the canonical renumbered form, not a temp alias, so the token + // pattern only matches "$". A literal dollar followed by a non-digit is preserved. + List in = Collections.singletonList("CLIENT FILTER BY price > $5.50 AND tag = '$abc'"); + assertEquals(Collections.singletonList("CLIENT FILTER BY price > $1.50 AND tag = '$abc'"), + new ExplainTextNormalizer().normalize(in)); + } + + @Test + public void testJsonNormalizerRenumbersTempAliasesAcrossFields() { + ObjectNode root = mapper.createObjectNode(); + // First-appearance order is field-insertion order: $7 → $1, $9 → $2, $7 reused → $1. + root.put("serverWhereFilter", "SERVER FILTER BY ATABLE.ORGANIZATION_ID IN ($7.$9)"); + root.put("clientFilterBy", "X = $7"); + new ExplainJsonNormalizer().normalize(root); + assertEquals("SERVER FILTER BY ATABLE.ORGANIZATION_ID IN ($1.$2)", + root.get("serverWhereFilter").asText()); + assertEquals("X = $1", root.get("clientFilterBy").asText()); + } + + @Test + public void testJsonNormalizerSharesAliasStateWithRhsRecursion() { + ObjectNode root = mapper.createObjectNode(); + root.put("serverWhereFilter", "X IN ($3)"); + ObjectNode rhs = mapper.createObjectNode(); + rhs.put("serverWhereFilter", "Y = $3 AND Z = $5"); + root.set("rhsJoinQueryExplainPlan", rhs); + new ExplainJsonNormalizer().normalize(root); + assertEquals("X IN ($1)", root.get("serverWhereFilter").asText()); + assertEquals("Y = $1 AND Z = $2", + root.get("rhsJoinQueryExplainPlan").get("serverWhereFilter").asText()); + } + + @Test + public void testTextNormalizerPreservesAllGrammar() { + List in = Arrays.asList("CLIENT PARALLEL 1-WAY RANGE SCAN OVER ATABLE ['a','b']", + " SERVER FILTER BY (X = 1 AND Y = 'z')", + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [Y]", "CLIENT MERGE SORT", + "CLIENT 3 ROW LIMIT"); + assertEquals(Arrays.asList("CLIENT PARALLEL -WAY RANGE SCAN OVER ATABLE ['a','b']", + " SERVER FILTER BY (X = 1 AND Y = 'z')", + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [Y]", "CLIENT MERGE SORT", + "CLIENT 3 ROW LIMIT"), new ExplainTextNormalizer().normalize(in)); + } + + @Test + public void testJsonNormalizerErasesClusterFields() { + ObjectNode root = mapper.createObjectNode(); + root.put("iteratorTypeAndScanSize", "PARALLEL 16-WAY"); + root.put("splitsChunk", 4); + root.put("scanEstimatedRows", 1234L); + root.put("scanEstimatedSizeInBytes", 9876L); + root.put("estimatedRows", 4321L); + root.put("estimatedSizeInBytes", 6789L); + root.put("estimateInfoTs", 1748880000000L); + root.put("numRegionLocationLookups", 7); + root.set("regionLocations", mapper.createArrayNode().add("anything")); + new ExplainJsonNormalizer().normalize(root); + assertEquals("PARALLEL -WAY", root.get("iteratorTypeAndScanSize").asText()); + assertTrue(root.get("splitsChunk").isNull()); + assertTrue(root.get("scanEstimatedRows").isNull()); + assertTrue(root.get("scanEstimatedSizeInBytes").isNull()); + assertTrue(root.get("estimatedRows").isNull()); + assertTrue(root.get("estimatedSizeInBytes").isNull()); + assertTrue(root.get("estimateInfoTs").isNull()); + assertTrue(root.get("regionLocations").isNull()); + assertEquals(0, root.get("numRegionLocationLookups").asInt()); + } + + @Test + public void testJsonNormalizerRecursesIntoLhsJoinQueryExplainPlan() { + ObjectNode root = mapper.createObjectNode(); + root.put("iteratorTypeAndScanSize", "PARALLEL 5-WAY"); + root.put("numRegionLocationLookups", 1); + ObjectNode lhs = mapper.createObjectNode(); + lhs.put("iteratorTypeAndScanSize", "PARALLEL 7-WAY"); + lhs.put("numRegionLocationLookups", 42); + lhs.set("regionLocations", mapper.createArrayNode().add(1)); + root.set("lhsJoinQueryExplainPlan", lhs); + new ExplainJsonNormalizer().normalize(root); + assertEquals("PARALLEL -WAY", root.get("iteratorTypeAndScanSize").asText()); + assertEquals(0, root.get("numRegionLocationLookups").asInt()); + JsonNode nestedLhs = root.get("lhsJoinQueryExplainPlan"); + assertEquals("PARALLEL -WAY", nestedLhs.get("iteratorTypeAndScanSize").asText()); + assertEquals(0, nestedLhs.get("numRegionLocationLookups").asInt()); + assertTrue(nestedLhs.get("regionLocations").isNull()); + } + + @Test + public void testJsonNormalizerSharesAliasStateWithLhsRecursion() { + ObjectNode root = mapper.createObjectNode(); + root.put("serverWhereFilter", "X IN ($3)"); + ObjectNode lhs = mapper.createObjectNode(); + lhs.put("serverWhereFilter", "Y = $3 AND Z = $5"); + root.set("lhsJoinQueryExplainPlan", lhs); + new ExplainJsonNormalizer().normalize(root); + assertEquals("X IN ($1)", root.get("serverWhereFilter").asText()); + assertEquals("Y = $1 AND Z = $2", + root.get("lhsJoinQueryExplainPlan").get("serverWhereFilter").asText()); + } + + @Test + public void testJsonNormalizerRecursesIntoRhsJoinQueryExplainPlan() { + ObjectNode root = mapper.createObjectNode(); + root.put("iteratorTypeAndScanSize", "PARALLEL 5-WAY"); + root.put("numRegionLocationLookups", 1); + ObjectNode rhs = mapper.createObjectNode(); + rhs.put("iteratorTypeAndScanSize", "PARALLEL 12-WAY"); + rhs.put("numRegionLocationLookups", 99); + rhs.set("regionLocations", mapper.createArrayNode().add(1)); + root.set("rhsJoinQueryExplainPlan", rhs); + new ExplainJsonNormalizer().normalize(root); + assertEquals("PARALLEL -WAY", root.get("iteratorTypeAndScanSize").asText()); + assertEquals(0, root.get("numRegionLocationLookups").asInt()); + JsonNode nestedRhs = root.get("rhsJoinQueryExplainPlan"); + assertEquals("PARALLEL -WAY", nestedRhs.get("iteratorTypeAndScanSize").asText()); + assertEquals(0, nestedRhs.get("numRegionLocationLookups").asInt()); + assertTrue(nestedRhs.get("regionLocations").isNull()); + } + + @Test + public void testJacksonFieldOrderMatchesPropertyOrderAnnotation() throws Exception { + // The serialized field order must exactly follow the @JsonPropertyOrder declaration. Deriving + // the expected order from the annotation keeps this test correct across future reorderings. + String[] expectedOrder = + ExplainPlanAttributes.class.getAnnotation(JsonPropertyOrder.class).value(); + String json = mapper.writeValueAsString(new ExplainPlanAttributesBuilder().build()); + int prevIdx = -1; + String prevName = null; + for (String name : expectedOrder) { + int idx = json.indexOf("\"" + name + "\""); + assertTrue(name + " present in serialized JSON", idx >= 0); + assertTrue(name + " must serialize after " + prevName, idx > prevIdx); + prevIdx = idx; + prevName = name; + } + } + + @Test + public void testRegionLocationsSerializerRendersTriple() throws Exception { + HRegionLocation loc = new HRegionLocation( + RegionInfoBuilder.newBuilder(TableName.valueOf("FOO")).setStartKey(new byte[] { 0x01, 0x02 }) + .setEndKey(new byte[] { 0x03, 0x04 }).build(), + ServerName.valueOf("rs.example.com", 16020, 1234567890L)); + ExplainPlanAttributes a = new ExplainPlanAttributesBuilder() + .setRegionLocations(Collections.singletonList(loc)).setNumRegionLocationLookups(1).build(); + JsonNode tree = mapper.readTree(mapper.writeValueAsString(a)); + JsonNode entry = tree.get("regionLocations").get(0); + assertEquals("\\x01\\x02", entry.get("startKey").asText()); + assertEquals("\\x03\\x04", entry.get("endKey").asText()); + assertTrue(entry.get("server").asText().contains("rs.example.com")); + } + + @Test + public void testServerMergeColumnsSerializerEmitsSortedNames() throws Exception { + Set cols = new HashSet<>( + Arrays.asList(column("CF", "B_COL"), column("CF", "A_COL"), column("CF", "C_COL"))); + ExplainPlanAttributes a = + new ExplainPlanAttributesBuilder().setServerMergeColumns(cols).build(); + JsonNode tree = mapper.readTree(mapper.writeValueAsString(a)); + JsonNode array = tree.get("serverMergeColumns"); + assertEquals(3, array.size()); + // PColumn.toString() uses QueryConstants.NAME_SEPARATOR (".") between family and name. + assertEquals("CF.A_COL", array.get(0).asText()); + assertEquals("CF.B_COL", array.get(1).asText()); + assertEquals("CF.C_COL", array.get(2).asText()); + } + + @Test + public void testDiffMessageShowsExpectedAndActualForTextMismatch() { + ExplainPlan plan = samplePlan("PARALLEL 1-WAY", "FULL SCAN "); + // Caller's "expected" disagrees with what the plan actually emits. + List divergentExpectedText = + text("CLIENT PARALLEL -WAY FULL SCAN OVER T", " SERVER FILTER BY (X = 9)"); + ObjectNode divergentExpectedJson = defaultAttrs() + .put("iteratorTypeAndScanSize", "PARALLEL -WAY").put("explainScanType", "FULL SCAN ") + .put("tableName", "T").put("serverWhereFilter", "SERVER FILTER BY (X = 9)"); + try { + new ExplainOracle().verify("x", plan, divergentExpectedText, divergentExpectedJson); + fail("Expected AssertionError for diverged plan"); + } catch (AssertionError expected) { + String msg = expected.getMessage(); + assertTrue(msg.contains("Text mismatch for case 'x'")); + assertTrue(msg.contains("SERVER PROJECTION FILTER BY FIRST KEY ONLY")); + assertTrue(msg.contains("SERVER FILTER BY (X = 9)")); + } catch (Exception e) { + fail("Unexpected exception type: " + e); + } + } + + private void verifyQuery(String caseId, String query, List expectedText, + JsonNode expectedJson) throws Exception { + verifyQuery(caseId, query, defaultProps(), expectedText, expectedJson); + } + + private void verifyQuery(String caseId, String query, Properties props, List expectedText, + JsonNode expectedJson) throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + ExplainPlan plan = ExplainPlanTestUtil.getExplainPlan(conn, query); + oracle.verify(caseId, plan, expectedText, expectedJson); + } + } + + private void verifyMutation(String caseId, String query, boolean autoCommit, + List expectedText, JsonNode expectedJson) throws Exception { + Properties props = defaultProps(); + if (autoCommit) { + props.setProperty(AUTO_COMMIT_ATTRIB, "true"); + } + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + ExplainPlan plan = compileMutation(conn, query); + oracle.verify(caseId, plan, expectedText, expectedJson); + } + } + + private ExplainPlan compileMutation(Connection conn, String query) throws SQLException { + return ExplainPlanTestUtil.getMutationExplainPlan(conn, query); + } + + /** Assert that the optimized plan-steps text for {@code query} contains {@code expectedLine}. */ + private static void assertPlanContainsLine(Connection conn, String query, String expectedLine) + throws SQLException { + List steps = ExplainPlanTestUtil.getPlanSteps(conn, query); + assertTrue("expected plan to contain line '" + expectedLine + "' but was " + steps, + steps.contains(expectedLine)); + } + + /** Assert that no plan-steps text line for {@code query} contains {@code needle}. */ + private static void assertNoPlanLineContains(Connection conn, String query, String needle) + throws SQLException { + List steps = ExplainPlanTestUtil.getPlanSteps(conn, query); + for (String s : steps) { + assertFalse( + "expected no plan line containing '" + needle + "' but found '" + s + "' in " + steps, + s.contains(needle)); + } + } + + private static Properties defaultProps() { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + props.setProperty(DATE_FORMAT_ATTRIB, "yyyy-MM-dd"); + return props; + } + + private static List text(String... lines) { + return Arrays.asList(lines); + } + + /** + * Returns a fresh {@link ObjectNode} populated with the JSON shape that + * {@link ExplainPlanAttributes#getDefaultExplainPlan()} serializes. + */ + private static ObjectNode defaultAttrs() { + ObjectNode n = mapper.createObjectNode(); + n.putNull("tenantId"); + n.putNull("viewName"); + n.putNull("viewBaseName"); + n.putNull("cdcScopes"); + n.putNull("txnProvider"); + n.putNull("rewrites"); + n.putNull("estimatedRows"); + n.putNull("estimatedSizeInBytes"); + n.putNull("estimateInfoTs"); + n.putNull("abstractExplainPlan"); + n.putNull("onDuplicateKeyAction"); + n.putNull("serverUpdateSet"); + n.put("returningRow", false); + n.putNull("splitsChunk"); + n.putNull("scanEstimatedRows"); + n.putNull("scanEstimatedSizeInBytes"); + n.putNull("iteratorTypeAndScanSize"); + n.putNull("samplingRate"); + n.put("useRoundRobinIterator", false); + n.putNull("hexStringRVCOffset"); + n.putNull("consistency"); + n.putNull("hint"); + n.putNull("serverSortedBy"); + n.putNull("explainScanType"); + n.putNull("tableName"); + n.putNull("keyRanges"); + n.putNull("indexName"); + n.putNull("indexKind"); + n.putNull("indexRule"); + n.putNull("functionalMatch"); + n.putNull("indexRejected"); + n.putNull("saltBuckets"); + n.putNull("regionsPlanned"); + n.putNull("scanTimeRangeMin"); + n.putNull("scanTimeRangeMax"); + n.putNull("serverWhereFilter"); + n.putNull("serverDistinctFilter"); + n.putNull("serverOffset"); + n.putNull("serverRowLimit"); + n.putNull("serverParsedProjections"); + n.putNull("serverProject"); + n.putNull("serverFilters"); + n.putNull("ignoredHints"); + n.put("serverFirstKeyOnlyProjection", false); + n.put("serverEmptyColumnOnlyProjection", false); + n.putNull("serverAggregate"); + n.putNull("clientFilterBy"); + n.putNull("clientFilters"); + n.putNull("clientAggregate"); + n.putNull("clientSortedBy"); + n.putNull("clientAfterAggregate"); + n.putNull("clientDistinctFilter"); + n.putNull("clientOffset"); + n.putNull("clientRowLimit"); + n.putNull("clientSequenceCount"); + n.putNull("clientCursorName"); + n.putNull("clientSortAlgo"); + n.putNull("clientSteps"); + n.putNull("lhsJoinQueryExplainPlan"); + n.putNull("rhsJoinQueryExplainPlan"); + n.putNull("serverMergeColumns"); + n.putNull("regionLocations"); + n.putNull("regionLocationsTotalSize"); + n.put("numRegionLocationLookups", 0); + n.putNull("subPlans"); + n.putNull("serverGroupByLimit"); + n.putNull("dynamicServerFilter"); + n.putNull("afterJoinFilter"); + n.putNull("joinScannerLimit"); + n.put("sortMergeSkipMerge", false); + return n; + } + + /** + * Convenience wrapper that builds {@link #defaultAttrs()} for scans. + * @param scanType the {@code explainScanType} string e.g. {@code "FULL SCAN"}) + * @param table the {@code tableName} value + * @param keys the {@code keyRanges} string (may be {@code null} or empty) + */ + private static ObjectNode scanAttrs(String scanType, String table, String keys) { + ObjectNode n = defaultAttrs(); + n.put("iteratorTypeAndScanSize", "PARALLEL -WAY"); + n.put("consistency", "STRONG"); + n.put("explainScanType", scanType.trim()); + n.put("tableName", table); + n.put("indexName", table); + n.put("indexRule", scanType.trim().startsWith("POINT LOOKUP") ? "point lookup" : "data table"); + if (keys != null) { + n.put("keyRanges", keys.trim()); + } + return n; + } + + /** A plain {@link ObjectNode} alias for clarity in tests that don't use {@link #scanAttrs}. */ + private static ObjectNode attrs() { + return defaultAttrs(); + } + + /** + * Convenience wrapper that builds {@link #defaultAttrs()} for an atomic UPSERT operator, + * populating {@code onDuplicateKeyAction} and the {@code serverUpdateSet} assignment array. + * @param action the {@code OnDuplicateKeyType} name (e.g. {@code "UPDATE"}) + * @param assignments the ordered {@code = } renderings + */ + private static ObjectNode onDupKeyAttrs(String action, String... assignments) { + ObjectNode n = defaultAttrs(); + n.put("onDuplicateKeyAction", action); + ArrayNode arr = n.putArray("serverUpdateSet"); + for (String a : assignments) { + arr.add(a); + } + return n; + } + + /** Build a {@code clientSteps} JSON array for embedding into an expected attributes object. */ + private static ArrayNode clientSteps(String... steps) { + ArrayNode arr = mapper.createArrayNode(); + for (String s : steps) { + arr.add(s); + } + return arr; + } + + /** Build a {@code rewrites} JSON array for embedding into an expected attributes object. */ + private static ArrayNode rewriteList(String... rewrites) { + ArrayNode arr = mapper.createArrayNode(); + for (String s : rewrites) { + arr.add(s); + } + return arr; + } + + private static ExplainPlan samplePlan(String way, String scanType) { + ExplainPlanAttributes a = new ExplainPlanAttributesBuilder().setIteratorTypeAndScanSize(way) + .setExplainScanType(scanType).setTableName("T").setServerFirstKeyOnlyProjection(true).build(); + return new ExplainPlan(Arrays.asList("CLIENT " + way + " " + scanType.trim() + " OVER T", + " SERVER PROJECTION FILTER BY FIRST KEY ONLY"), a); + } + + private static PColumn column(String family, String name) { + PName fName = PNameFactory.newName(family); + PName cName = PNameFactory.newName(name); + return new PColumnImpl(cName, fName, PInteger.INSTANCE, null, null, false, 0, SortOrder.ASC, 0, + null, false, "expression", false, false, name.getBytes(), 0L); + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java new file mode 100644 index 00000000000..135180db837 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java @@ -0,0 +1,999 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.query.explain; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.phoenix.compile.ExplainPlan; +import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.optimize.OptimizerReasons; +import org.apache.phoenix.optimize.RejectedIndexEntry; +import org.apache.phoenix.parse.ExplainOptions; +import org.apache.phoenix.parse.UpsertStatement.OnDuplicateKeyType; + +/** + * Test helpers for retrieving the {@link ExplainPlan} and its structured + * {@link ExplainPlanAttributes} for a query or mutation (without going through the textual + * {@code EXPLAIN ...} ResultSet path), plus a fluent {@link ExplainPlanAssert} API for asserting on + * the attribute values. + */ +public final class ExplainPlanTestUtil { + + private ExplainPlanTestUtil() { + } + + /** Optimize {@code query} and return its {@link ExplainPlan}. */ + public static ExplainPlan getExplainPlan(Connection conn, String query) throws SQLException { + try (PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class)) { + return statement.optimizeQuery().getExplainPlan(); + } + } + + /** Optimize {@code query} and return its structured {@link ExplainPlanAttributes}. */ + public static ExplainPlanAttributes getExplainAttributes(Connection conn, String query) + throws SQLException { + return getExplainPlan(conn, query).getPlanStepsAsAttributes(); + } + + /** + * Optimize {@code query} and return its structured {@link ExplainPlanAttributes} with the given + * {@link ExplainOptions} applied to the plan's {@code StatementContext}. Used to exercise region + * location information, which is only populated when the {@code REGIONS} option is requested. + */ + public static ExplainPlanAttributes getExplainAttributes(Connection conn, String query, + ExplainOptions options) throws SQLException { + try (PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class)) { + QueryPlan plan = statement.optimizeQuery(); + plan.getContext().setExplainOptions(options); + return plan.getExplainPlan().getPlanStepsAsAttributes(); + } + } + + /** Optimize {@code query} and return its plan-steps text. */ + public static List getPlanSteps(Connection conn, String query) throws SQLException { + return getExplainPlan(conn, query).getPlanSteps(); + } + + /** + * Optimize {@code query} with the given {@link ExplainOptions} applied to the plan's + * {@code StatementContext} and return its plan-steps text. + */ + public static List getPlanSteps(Connection conn, String query, ExplainOptions options) + throws SQLException { + try (PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class)) { + QueryPlan plan = statement.optimizeQuery(); + plan.getContext().setExplainOptions(options); + return plan.getExplainPlan().getPlanSteps(); + } + } + + /** Compile a mutation (UPSERT/DELETE) and return its {@link ExplainPlan}. */ + public static ExplainPlan getMutationExplainPlan(Connection conn, String query) + throws SQLException { + try (PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class)) { + return statement.compileMutation().getExplainPlan(); + } + } + + /** Compile a mutation (UPSERT/DELETE) and return its structured {@link ExplainPlanAttributes}. */ + public static ExplainPlanAttributes getMutationExplainAttributes(Connection conn, String query) + throws SQLException { + return getMutationExplainPlan(conn, query).getPlanStepsAsAttributes(); + } + + /** Begin assertions on the given attributes. */ + public static ExplainPlanAssert assertPlan(ExplainPlanAttributes attributes) { + assertNotNull("ExplainPlanAttributes must not be null", attributes); + return new ExplainPlanAssert(attributes, null, "plan"); + } + + /** Optimize {@code query} on {@code conn} and begin assertions on its plan attributes. */ + public static ExplainPlanAssert assertPlan(Connection conn, String query) throws SQLException { + return assertPlan(getExplainAttributes(conn, query)); + } + + /** + * Optimize {@code query} on {@code conn} with the {@code REGIONS} option enabled and begin + * assertions on its plan attributes. Use this instead of {@link #assertPlan(Connection, String)} + * when asserting on region-location attributes, which are only populated when {@code REGIONS} is + * requested. + */ + public static ExplainPlanAssert assertPlanWithRegions(Connection conn, String query) + throws SQLException { + return assertPlan(getExplainAttributes(conn, query, ExplainOptions.WITH_REGIONS)); + } + + /** + * Optimize {@code query} on {@code conn} with the {@code VERBOSE} option enabled and begin + * assertions on its plan attributes. Use this when asserting on VERBOSE-only attributes such as + * {@code serverProject}, {@code serverFilters}, and {@code ignoredHints}. + */ + public static ExplainPlanAssert assertPlanWithVerbose(Connection conn, String query) + throws SQLException { + return assertPlan(getExplainAttributes(conn, query, ExplainOptions.VERBOSE)); + } + + /** + * Optimize an already-prepared and, if needed, parameter-bound {@link PhoenixPreparedStatement} + * and begin assertions on its plan attributes. + */ + public static ExplainPlanAssert assertPlan(PhoenixPreparedStatement statement) + throws SQLException { + return assertPlan(statement.optimizeQuery().getExplainPlan().getPlanStepsAsAttributes()); + } + + /** Begin assertions on the plan attributes of a resolved {@link QueryPlan}. */ + public static ExplainPlanAssert assertPlan(QueryPlan queryPlan) throws SQLException { + return assertPlan(queryPlan.getExplainPlan().getPlanStepsAsAttributes()); + } + + /** Compile the mutation {@code query} on {@code conn} and begin assertions on its attributes. */ + public static ExplainPlanAssert assertMutationPlan(Connection conn, String query) + throws SQLException { + return assertPlan(getMutationExplainAttributes(conn, query)); + } + + /** + * Compile an already-prepared and, if needed, parameter-bound mutation + * {@link PhoenixPreparedStatement} and begin assertions on its attributes. + */ + public static ExplainPlanAssert assertMutationPlan(PhoenixPreparedStatement statement) + throws SQLException { + return assertPlan(statement.compileMutation().getExplainPlan().getPlanStepsAsAttributes()); + } + + /** Fluent assertions over {@link ExplainPlanAttributes}. */ + public static final class ExplainPlanAssert { + + private final ExplainPlanAttributes attributes; + private final ExplainPlanAssert parent; + private final String context; + + private ExplainPlanAssert(ExplainPlanAttributes attributes, ExplainPlanAssert parent, + String context) { + this.attributes = attributes; + this.parent = parent; + this.context = context; + } + + public ExplainPlanAttributes attributes() { + return attributes; + } + + /** + * Assert the scan type, e.g. {@code "FULL SCAN"}, {@code "RANGE SCAN"}, + * {@code "POINT LOOKUP ON 1 KEY"}, {@code "SKIP SCAN ON 3 KEYS"}. + */ + public ExplainPlanAssert scanType(String expected) { + assertEquals(at("explainScanType"), trim(expected), trim(attributes.getExplainScanType())); + return this; + } + + /** + * Assert the scan type starts with {@code prefix}, useful for variable width prefixes such as + * {@code "POINT LOOKUP ON "} where the number of keys is data dependent. + */ + public ExplainPlanAssert scanTypeStartsWith(String prefix) { + String actual = attributes.getExplainScanType(); + assertNotNull(at("explainScanType") + " must not be null", actual); + assertTrue( + at("explainScanType") + " expected to start with '" + prefix + "' but was '" + actual + "'", + actual.startsWith(prefix)); + return this; + } + + /** Assert the scanned table (or index) name. */ + public ExplainPlanAssert table(String expected) { + assertEquals(at("tableName"), expected, attributes.getTableName()); + return this; + } + + /** + * Assert the scanned table or index name contains {@code expected}. Useful for "index used" + * checks where the exact name carries a suffix. + */ + public ExplainPlanAssert tableContains(String expected) { + String actual = attributes.getTableName(); + assertNotNull(at("tableName") + " must not be null", actual); + assertTrue( + at("tableName") + " expected to contain '" + expected + "' but was '" + actual + "'", + actual.contains(expected)); + return this; + } + + /** Assert the chosen per-scan index (or data table) name. */ + public ExplainPlanAssert indexName(String expected) { + assertEquals(at("indexName"), expected, attributes.getIndexName()); + return this; + } + + /** + * Assert the per-scan index kind token: {@code "LOCAL"}, {@code "GLOBAL"}, or + * {@code "UNCOVERED GLOBAL"}. + */ + public ExplainPlanAssert indexKind(String expected) { + assertEquals(at("indexKind"), expected, attributes.getIndexKind()); + return this; + } + + /** + * Assert the optimizer's index-selection rule label, e.g. one of the + * {@code OptimizerReasons.RULE_*} constants. + */ + public ExplainPlanAssert indexRule(String expected) { + assertEquals(at("indexRule"), expected, attributes.getIndexRule()); + return this; + } + + /** + * Assert the optimizer's separate functional-index match disclosure equals + * {@code "matches "} (see {@link OptimizerReasons#matches(String)}). The selection + * {@code indexRule} is disclosed independently and is asserted with {@link #indexRule(String)}. + */ + public ExplainPlanAssert functionalMatch(String expression) { + assertEquals(at("functionalMatch"), OptimizerReasons.matches(expression), + attributes.getFunctionalMatch()); + return this; + } + + /** Assert no functional-index match disclosure was recorded for this plan. */ + public ExplainPlanAssert functionalMatchNone() { + assertEquals(at("functionalMatch"), null, attributes.getFunctionalMatch()); + return this; + } + + /** Assert the number of rejected index candidates recorded for this plan. */ + public ExplainPlanAssert indexRejectedCount(int expected) { + List rejected = attributes.getIndexRejected(); + int actual = rejected == null ? 0 : rejected.size(); + assertEquals(at("indexRejected.size"), expected, actual); + return this; + } + + /** Assert the i-th rejected index candidate's name and reason. */ + public ExplainPlanAssert indexRejected(int i, String name, String reason) { + List rejected = attributes.getIndexRejected(); + assertNotNull(at("indexRejected") + " must not be null", rejected); + assertTrue(at("indexRejected") + " has no index " + i + " (size=" + rejected.size() + ")", + i >= 0 && i < rejected.size()); + RejectedIndexEntry entry = rejected.get(i); + assertEquals(at("indexRejected[" + i + "].name"), name, entry.getName()); + assertEquals(at("indexRejected[" + i + "].reason"), reason, entry.getReason()); + return this; + } + + /** Assert that no index candidates were rejected (null or empty list). */ + public ExplainPlanAssert indexRejectedNone() { + List rejected = attributes.getIndexRejected(); + assertTrue(at("indexRejected") + " expected none but was " + rejected, + rejected == null || rejected.isEmpty()); + return this; + } + + /** + * Assert that the rejected index list contains an entry with {@code name} and {@code reason}. + */ + public ExplainPlanAssert indexRejectedContains(String name, String reason) { + List rejected = attributes.getIndexRejected(); + assertNotNull(at("indexRejected") + " must not be null", rejected); + for (RejectedIndexEntry entry : rejected) { + if (entry.getName().equals(name) && entry.getReason().equals(reason)) { + return this; + } + } + throw new AssertionError(at("indexRejected") + " expected to contain {" + name + ", " + reason + + "} but was " + rejected); + } + + /** Assert the salt bucket count of the scanned table (null when not salted). */ + public ExplainPlanAssert saltBuckets(Integer expected) { + assertEquals(at("saltBuckets"), expected, attributes.getSaltBuckets()); + return this; + } + + /** Assert the number of regions the scan is planned to hit (null when unknown). */ + public ExplainPlanAssert regionsPlanned(Integer expected) { + assertEquals(at("regionsPlanned"), expected, attributes.getRegionsPlanned()); + return this; + } + + /** Assert the hex-string row-value-constructor offset marker. */ + public ExplainPlanAssert hexStringRVCOffset(String expected) { + assertEquals(at("hexStringRVCOffset"), expected, attributes.getHexStringRVCOffset()); + return this; + } + + /** Assert the key ranges string (exact; leading space is significant). */ + public ExplainPlanAssert keyRanges(String expected) { + assertEquals(at("keyRanges"), expected, attributes.getKeyRanges()); + return this; + } + + public ExplainPlanAssert abstractExplainPlan(String expected) { + assertEquals(at("abstractExplainPlan"), expected, attributes.getAbstractExplainPlan()); + return this; + } + + /** + * Assert the {@code ON DUPLICATE KEY} flavor disclosed on an atomic UPSERT mutation operator. + */ + public ExplainPlanAssert onDuplicateKeyAction(OnDuplicateKeyType expected) { + assertEquals(at("onDuplicateKeyAction"), expected, attributes.getOnDuplicateKeyAction()); + return this; + } + + /** + * Assert the ordered {@code = } assignments disclosed under + * {@code ON DUPLICATE KEY UPDATE}. + */ + public ExplainPlanAssert serverUpdateSet(String... expected) { + assertEquals(at("serverUpdateSet"), Arrays.asList(expected), attributes.getServerUpdateSet()); + return this; + } + + /** Assert the number of {@code SERVER UPDATE SET} assignments. */ + public ExplainPlanAssert serverUpdateSetCount(int expected) { + List actual = attributes.getServerUpdateSet(); + int actualCount = actual == null ? 0 : actual.size(); + assertEquals(at("serverUpdateSet.size"), expected, actualCount); + return this; + } + + /** Assert the mutation operator discloses {@code RETURNING *}. */ + public ExplainPlanAssert returningRow() { + assertTrue(at("returningRow") + " expected true", attributes.isReturningRow()); + return this; + } + + /** Assert the mutation operator does not disclose {@code RETURNING *}. */ + public ExplainPlanAssert noReturningRow() { + assertTrue(at("returningRow") + " expected false", !attributes.isReturningRow()); + return this; + } + + /** Assert the read consistency level. */ + public ExplainPlanAssert consistency(String expected) { + assertEquals(at("consistency"), expected, + attributes.getConsistency() == null ? null : attributes.getConsistency().name()); + return this; + } + + public ExplainPlanAssert serverWhereFilter(String expected) { + assertEquals(at("serverWhereFilter"), expected, attributes.getServerWhereFilter()); + return this; + } + + /** Assert that {@code serverWhereFilter} is equal to one of the {@code allowed} values. */ + public ExplainPlanAssert serverWhereFilterAnyOf(String... allowed) { + String actual = attributes.getServerWhereFilter(); + for (String s : allowed) { + if (s == null ? actual == null : s.equals(actual)) { + return this; + } + } + throw new AssertionError(at("serverWhereFilter") + " expected one of " + + Arrays.toString(allowed) + " but was " + actual); + } + + public ExplainPlanAssert serverAggregate(String expected) { + assertEquals(at("serverAggregate"), expected, attributes.getServerAggregate()); + return this; + } + + public ExplainPlanAssert serverSortedBy(String expected) { + assertEquals(at("serverSortedBy"), expected, attributes.getServerSortedBy()); + return this; + } + + public ExplainPlanAssert serverDistinctFilter(String expected) { + assertEquals(at("serverDistinctFilter"), expected, attributes.getServerDistinctFilter()); + return this; + } + + public ExplainPlanAssert serverMergeColumns(String expected) { + assertEquals(at("serverMergeColumns"), expected, + attributes.getServerMergeColumns() == null + ? null + : attributes.getServerMergeColumns().toString()); + return this; + } + + public ExplainPlanAssert clientFilterBy(String expected) { + assertEquals(at("clientFilterBy"), expected, attributes.getClientFilterBy()); + return this; + } + + public ExplainPlanAssert clientAggregate(String expected) { + assertEquals(at("clientAggregate"), expected, attributes.getClientAggregate()); + return this; + } + + public ExplainPlanAssert clientSortedBy(String expected) { + assertEquals(at("clientSortedBy"), expected, attributes.getClientSortedBy()); + return this; + } + + public ExplainPlanAssert clientAfterAggregate(String expected) { + assertEquals(at("clientAfterAggregate"), expected, attributes.getClientAfterAggregate()); + return this; + } + + public ExplainPlanAssert clientDistinctFilter(String expected) { + assertEquals(at("clientDistinctFilter"), expected, attributes.getClientDistinctFilter()); + return this; + } + + public ExplainPlanAssert clientSortAlgo(String expected) { + assertEquals(at("clientSortAlgo"), expected, attributes.getClientSortAlgo()); + return this; + } + + public ExplainPlanAssert serverRowLimit(Long expected) { + assertEquals(at("serverRowLimit"), expected, attributes.getServerRowLimit()); + return this; + } + + public ExplainPlanAssert serverGroupByLimit(Integer expected) { + assertEquals(at("serverGroupByLimit"), expected, attributes.getServerGroupByLimit()); + return this; + } + + public ExplainPlanAssert clientRowLimit(Integer expected) { + assertEquals(at("clientRowLimit"), expected, attributes.getClientRowLimit()); + return this; + } + + public ExplainPlanAssert serverOffset(Integer expected) { + assertEquals(at("serverOffset"), expected, attributes.getServerOffset()); + return this; + } + + public ExplainPlanAssert clientOffset(Integer expected) { + assertEquals(at("clientOffset"), expected, attributes.getClientOffset()); + return this; + } + + public ExplainPlanAssert clientSequenceCount(Integer expected) { + assertEquals(at("clientSequenceCount"), expected, attributes.getClientSequenceCount()); + return this; + } + + public ExplainPlanAssert hint(String expected) { + assertEquals(at("hint"), expected, + attributes.getHint() == null ? null : attributes.getHint().toString()); + return this; + } + + public ExplainPlanAssert samplingRate(Double expected) { + assertEquals(at("samplingRate"), expected, attributes.getSamplingRate()); + return this; + } + + /** Assert the entire server-parsed-projection map matches {@code expected}. */ + public ExplainPlanAssert serverParsedProjections(Map> expected) { + assertEquals(at("serverParsedProjections"), expected, + attributes.getServerParsedProjections()); + return this; + } + + /** + * Assert that the named bucket ({@code ARRAY}, {@code JSON}, {@code BSON}) holds exactly the + * listed per-expression renderings, in order. + */ + public ExplainPlanAssert serverParsedProjections(String label, String... expected) { + Map> actual = attributes.getServerParsedProjections(); + assertNotNull(at("serverParsedProjections") + " must not be null", actual); + List bucket = actual.get(label); + assertNotNull(at("serverParsedProjections[" + label + "]") + " must not be null", bucket); + assertEquals(at("serverParsedProjections[" + label + "]"), Arrays.asList(expected), bucket); + return this; + } + + /** Assert that no server-parsed projections were disclosed (null or empty). */ + public ExplainPlanAssert serverParsedProjectionsNone() { + Map> actual = attributes.getServerParsedProjections(); + assertTrue(at("serverParsedProjections") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + + /** Assert the number of expressions in the named bucket. */ + public ExplainPlanAssert serverParsedProjectionCount(String label, int expected) { + Map> actual = attributes.getServerParsedProjections(); + List bucket = actual == null ? null : actual.get(label); + int actualCount = bucket == null ? 0 : bucket.size(); + assertEquals(at("serverParsedProjections[" + label + "].size"), expected, actualCount); + return this; + } + + /** Assert the entire VERBOSE {@code PROJECT} column list matches {@code expected}, in order. */ + public ExplainPlanAssert serverProject(String... expected) { + List actual = attributes.getServerProject(); + List actualOrEmpty = actual == null ? Collections. emptyList() : actual; + assertEquals(at("serverProject"), Arrays.asList(expected), actualOrEmpty); + return this; + } + + /** Assert the number of VERBOSE {@code PROJECT} columns. */ + public ExplainPlanAssert serverProjectCount(int expected) { + List actual = attributes.getServerProject(); + int actualCount = actual == null ? 0 : actual.size(); + assertEquals(at("serverProject.size"), expected, actualCount); + return this; + } + + /** Assert that no VERBOSE {@code PROJECT} disclosure was emitted (null or empty). */ + public ExplainPlanAssert serverProjectNone() { + List actual = attributes.getServerProject(); + assertTrue(at("serverProject") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + + /** Assert the number of VERBOSE server filter predicates. */ + public ExplainPlanAssert serverFilterCount(int expected) { + List actual = attributes.getServerFilters(); + int actualCount = actual == null ? 0 : actual.size(); + assertEquals(at("serverFilters.size"), expected, actualCount); + return this; + } + + /** Assert that no VERBOSE server filter breakdown was emitted (null or empty). */ + public ExplainPlanAssert serverFiltersNone() { + List actual = attributes.getServerFilters(); + assertTrue(at("serverFilters") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + + /** Assert the i-th VERBOSE server filter's rendered expression. */ + public ExplainPlanAssert serverFilter(int i, String expectedExpr) { + ExplainPlanAttributes.ExplainFilter filter = serverFilterAt(i); + assertEquals(at("serverFilters[" + i + "].expr"), expectedExpr, filter.getExpr()); + return this; + } + + /** Assert the i-th VERBOSE server filter's origin attribution, in order. */ + public ExplainPlanAssert serverFilterOrigin(int i, String... expectedOrigin) { + ExplainPlanAttributes.ExplainFilter filter = serverFilterAt(i); + List actual = + filter.getOrigin() == null ? Collections. emptyList() : filter.getOrigin(); + assertEquals(at("serverFilters[" + i + "].origin"), Arrays.asList(expectedOrigin), actual); + return this; + } + + /** Assert the i-th VERBOSE server filter's path-test sub-tag (or {@code null}). */ + public ExplainPlanAssert serverFilterPathTest(int i, String expectedSubtag) { + ExplainPlanAttributes.ExplainFilter filter = serverFilterAt(i); + assertEquals(at("serverFilters[" + i + "].pathTestSubtag"), expectedSubtag, + filter.getPathTestSubtag()); + return this; + } + + private ExplainPlanAttributes.ExplainFilter serverFilterAt(int i) { + List filters = attributes.getServerFilters(); + assertNotNull(at("serverFilters") + " must not be null", filters); + assertTrue(at("serverFilters") + " has no index " + i + " (size=" + filters.size() + ")", + i >= 0 && i < filters.size()); + return filters.get(i); + } + + /** Assert the number of VERBOSE client filter predicates. */ + public ExplainPlanAssert clientFilterCount(int expected) { + List actual = attributes.getClientFilters(); + int actualCount = actual == null ? 0 : actual.size(); + assertEquals(at("clientFilters.size"), expected, actualCount); + return this; + } + + /** Assert that no VERBOSE client filter breakdown was emitted (null or empty). */ + public ExplainPlanAssert clientFiltersNone() { + List actual = attributes.getClientFilters(); + assertTrue(at("clientFilters") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + + /** Assert the i-th VERBOSE client filter's rendered expression. */ + public ExplainPlanAssert clientFilter(int i, String expectedExpr) { + ExplainPlanAttributes.ExplainFilter filter = clientFilterAt(i); + assertEquals(at("clientFilters[" + i + "].expr"), expectedExpr, filter.getExpr()); + return this; + } + + /** Assert the i-th VERBOSE client filter's origin attribution, in order. */ + public ExplainPlanAssert clientFilterOrigin(int i, String... expectedOrigin) { + ExplainPlanAttributes.ExplainFilter filter = clientFilterAt(i); + List actual = + filter.getOrigin() == null ? Collections. emptyList() : filter.getOrigin(); + assertEquals(at("clientFilters[" + i + "].origin"), Arrays.asList(expectedOrigin), actual); + return this; + } + + /** Assert the i-th VERBOSE client filter's path-test sub-tag (or {@code null}). */ + public ExplainPlanAssert clientFilterPathTest(int i, String expectedSubtag) { + ExplainPlanAttributes.ExplainFilter filter = clientFilterAt(i); + assertEquals(at("clientFilters[" + i + "].pathTestSubtag"), expectedSubtag, + filter.getPathTestSubtag()); + return this; + } + + private ExplainPlanAttributes.ExplainFilter clientFilterAt(int i) { + List filters = attributes.getClientFilters(); + assertNotNull(at("clientFilters") + " must not be null", filters); + assertTrue(at("clientFilters") + " has no index " + i + " (size=" + filters.size() + ")", + i >= 0 && i < filters.size()); + return filters.get(i); + } + + /** Assert the entire VERBOSE ignored-hint map matches {@code expected}. */ + public ExplainPlanAssert ignoredHints(Map expected) { + assertEquals(at("ignoredHints"), expected, attributes.getIgnoredHints()); + return this; + } + + /** Assert the ignored-hint map carries {@code hint} mapped to {@code reason}. */ + public ExplainPlanAssert ignoredHint(String hint, String reason) { + Map actual = attributes.getIgnoredHints(); + assertNotNull(at("ignoredHints") + " must not be null", actual); + assertEquals(at("ignoredHints[" + hint + "]"), reason, actual.get(hint)); + return this; + } + + /** Assert that the ignored-hint map contains an entry for {@code hint}. */ + public ExplainPlanAssert hasIgnoredHint(String hint) { + Map actual = attributes.getIgnoredHints(); + assertTrue(at("ignoredHints") + " expected to contain '" + hint + "' but was " + actual, + actual != null && actual.containsKey(hint)); + return this; + } + + /** Assert that no ignored-hint disclosure was emitted (null or empty). */ + public ExplainPlanAssert ignoredHintsNone() { + Map actual = attributes.getIgnoredHints(); + assertTrue(at("ignoredHints") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + + public ExplainPlanAssert serverFirstKeyOnlyProjection(boolean expected) { + assertEquals(at("serverFirstKeyOnlyProjection"), expected, + attributes.isServerFirstKeyOnlyProjection()); + return this; + } + + /** + * Assert when {@code firstKeyOnly} is true the scan carries the {@code FIRST KEY ONLY} + * projection, otherwise it carries the {@code EMPTY COLUMN ONLY} projection. Convenience for + * callers that pick the expected kind from a flag. + */ + public ExplainPlanAssert serverProjectionFilter(boolean firstKeyOnly) { + return firstKeyOnly + ? serverFirstKeyOnlyProjection(true) + : serverEmptyColumnOnlyProjection(true); + } + + /** + * Assert either the {@code FIRST KEY ONLY} or the {@code EMPTY COLUMN ONLY} projection + * optimization is present. + */ + public ExplainPlanAssert serverProjectionFilterAnyOf() { + assertTrue( + at("serverFirstKeyOnlyProjection|serverEmptyColumnOnlyProjection") + + " expected one to be true", + attributes.isServerFirstKeyOnlyProjection() + || attributes.isServerEmptyColumnOnlyProjection()); + return this; + } + + public ExplainPlanAssert serverEmptyColumnOnlyProjection(boolean expected) { + assertEquals(at("serverEmptyColumnOnlyProjection"), expected, + attributes.isServerEmptyColumnOnlyProjection()); + return this; + } + + public ExplainPlanAssert useRoundRobinIterator(boolean expected) { + assertEquals(at("useRoundRobinIterator"), expected, attributes.isUseRoundRobinIterator()); + return this; + } + + public ExplainPlanAssert scanEstimatedRows(Long expected) { + assertEquals(at("scanEstimatedRows"), expected, attributes.getScanEstimatedRows()); + return this; + } + + public ExplainPlanAssert scanEstimatedBytes(Long expected) { + assertEquals(at("scanEstimatedSizeInBytes"), expected, + attributes.getScanEstimatedSizeInBytes()); + return this; + } + + public ExplainPlanAssert estimatedRows(Long expected) { + assertEquals(at("estimatedRows"), expected, attributes.getEstimatedRows()); + return this; + } + + public ExplainPlanAssert estimatedBytes(Long expected) { + assertEquals(at("estimatedSizeInBytes"), expected, attributes.getEstimatedSizeInBytes()); + return this; + } + + public ExplainPlanAssert estimateInfoTs(Long expected) { + assertEquals(at("estimateInfoTs"), expected, attributes.getEstimateInfoTs()); + return this; + } + + public ExplainPlanAssert splitsChunk(Integer expected) { + assertEquals(at("splitsChunk"), expected, attributes.getSplitsChunk()); + return this; + } + + public ExplainPlanAssert regionLocationCount(int expected) { + int actual = + attributes.getRegionLocations() == null ? 0 : attributes.getRegionLocations().size(); + assertEquals(at("regionLocations.size"), expected, actual); + return this; + } + + /** Assert that the EXPLAIN plan emitted at least one region location. */ + public ExplainPlanAssert regionLocationsNotEmpty() { + assertNotNull(at("regionLocations") + " must not be null", attributes.getRegionLocations()); + assertTrue(at("regionLocations") + " must not be empty", + !attributes.getRegionLocations().isEmpty()); + return this; + } + + /** + * Assert the total distinct region locations seen before any trimming applied to + * {@code regionLocations}. When the trim limit was not reached, this equals + * {@link #regionLocationCount(int)}. + */ + public ExplainPlanAssert regionLocationsTotalSize(Integer expected) { + assertEquals(at("regionLocationsTotalSize"), expected, + attributes.getRegionLocationsTotalSize()); + return this; + } + + /** Assert the number of region location lookups performed by the planner. */ + public ExplainPlanAssert numRegionLocationLookups(int expected) { + assertEquals(at("numRegionLocationLookups"), expected, + attributes.getNumRegionLocationLookups()); + return this; + } + + public ExplainPlanAssert dynamicServerFilter(String expected) { + assertEquals(at("dynamicServerFilter"), expected, attributes.getDynamicServerFilter()); + return this; + } + + public ExplainPlanAssert afterJoinFilter(String expected) { + assertEquals(at("afterJoinFilter"), expected, attributes.getAfterJoinFilter()); + return this; + } + + public ExplainPlanAssert joinScannerLimit(Long expected) { + assertEquals(at("joinScannerLimit"), expected, attributes.getJoinScannerLimit()); + return this; + } + + /** Assert the sort-merge-join "(SKIP MERGE)" marker on this (left) side of the join. */ + public ExplainPlanAssert sortMergeSkipMerge(boolean expected) { + assertEquals(at("sortMergeSkipMerge"), expected, attributes.isSortMergeSkipMerge()); + return this; + } + + /** Assert the (normalized) parallelism width, e.g. {@code "PARALLEL"} or {@code "SERIAL"}. */ + public ExplainPlanAssert iteratorType(String expectedPrefix) { + String iter = attributes.getIteratorTypeAndScanSize(); + assertNotNull(at("iteratorTypeAndScanSize"), iter); + assertTrue(at("iteratorTypeAndScanSize") + " expected to start with '" + expectedPrefix + + "' but was '" + iter + "'", iter.startsWith(expectedPrefix)); + return this; + } + + /** Navigate to the left-hand side plan (sort-merge join). */ + public ExplainPlanAssert lhs() { + ExplainPlanAttributes lhs = attributes.getLhsJoinQueryExplainPlan(); + assertNotNull(at("lhsJoinQueryExplainPlan") + " must not be null", lhs); + return new ExplainPlanAssert(lhs, this, context + ".lhs"); + } + + /** Navigate to the right-hand side plan (sort-merge join / union all). */ + public ExplainPlanAssert rhs() { + ExplainPlanAttributes rhs = attributes.getRhsJoinQueryExplainPlan(); + assertNotNull(at("rhsJoinQueryExplainPlan") + " must not be null", rhs); + return new ExplainPlanAssert(rhs, this, context + ".rhs"); + } + + /** Assert the number of ordered client-side pipeline steps on this node. */ + public ExplainPlanAssert clientStepCount(int expected) { + List steps = attributes.getClientSteps(); + int actual = steps == null ? 0 : steps.size(); + assertEquals(at("clientSteps.size"), expected, actual); + return this; + } + + /** Assert the i-th ordered client-side pipeline step on this node. */ + public ExplainPlanAssert clientStep(int i, String expected) { + List steps = attributes.getClientSteps(); + assertNotNull(at("clientSteps") + " must not be null", steps); + assertTrue(at("clientSteps") + " has no index " + i + " (size=" + steps.size() + ")", + i >= 0 && i < steps.size()); + assertEquals(at("clientSteps[" + i + "]"), expected, steps.get(i)); + return this; + } + + /** Assert the entire ordered client-side pipeline on this node matches {@code expected}. */ + public ExplainPlanAssert clientSteps(String... expected) { + List actual = attributes.getClientSteps(); + List actualOrEmpty = actual == null ? Collections. emptyList() : actual; + assertEquals(at("clientSteps"), Arrays.asList(expected), actualOrEmpty); + return this; + } + + /** Assert the number of hash-join sub-plans (children). */ + public ExplainPlanAssert subPlanCount(int expected) { + List subPlans = attributes.getSubPlans(); + int actual = subPlans == null ? 0 : subPlans.size(); + assertEquals(at("subPlans.size"), expected, actual); + return this; + } + + /** Navigate to the i-th hash-join sub-plan (child). */ + public ExplainPlanAssert subPlan(int i) { + List subPlans = attributes.getSubPlans(); + assertNotNull(at("subPlans") + " must not be null", subPlans); + assertTrue(at("subPlans") + " has no index " + i + " (size=" + subPlans.size() + ")", + i >= 0 && i < subPlans.size()); + return new ExplainPlanAssert(subPlans.get(i), this, context + ".subPlan[" + i + "]"); + } + + /** Assert the tenant id disclosed at the top of the plan. */ + public ExplainPlanAssert tenant(String expected) { + assertEquals(at("tenantId"), expected, attributes.getTenantId()); + return this; + } + + /** Assert that no tenant id disclosure was emitted. */ + public ExplainPlanAssert tenantNone() { + assertNull(at("tenantId") + " expected none but was " + attributes.getTenantId(), + attributes.getTenantId()); + return this; + } + + /** Assert the view name and its base (physical) table name disclosed at the top of the plan. */ + public ExplainPlanAssert view(String expectedName, String expectedBaseName) { + assertEquals(at("viewName"), expectedName, attributes.getViewName()); + assertEquals(at("viewBaseName"), expectedBaseName, attributes.getViewBaseName()); + return this; + } + + /** Assert the view name disclosed at the top of the plan. */ + public ExplainPlanAssert viewName(String expected) { + assertEquals(at("viewName"), expected, attributes.getViewName()); + return this; + } + + /** Assert that no view disclosure was emitted. */ + public ExplainPlanAssert viewNone() { + assertNull(at("viewName") + " expected none but was " + attributes.getViewName(), + attributes.getViewName()); + return this; + } + + /** Assert the CDC change scopes disclosed at the top of the plan. */ + public ExplainPlanAssert cdcScopes(String expected) { + assertEquals(at("cdcScopes"), expected, attributes.getCdcScopes()); + return this; + } + + /** Assert that no CDC scope disclosure was emitted. */ + public ExplainPlanAssert cdcScopesNone() { + assertNull(at("cdcScopes") + " expected none but was " + attributes.getCdcScopes(), + attributes.getCdcScopes()); + return this; + } + + /** Assert the transaction provider disclosed at the top of the plan. */ + public ExplainPlanAssert txnProvider(String expected) { + assertEquals(at("txnProvider"), expected, attributes.getTxnProvider()); + return this; + } + + /** Assert that no transaction provider disclosure was emitted. */ + public ExplainPlanAssert txnProviderNone() { + assertNull(at("txnProvider") + " expected none but was " + attributes.getTxnProvider(), + attributes.getTxnProvider()); + return this; + } + + /** Assert the number of distinct rewrite breadcrumbs disclosed at the top of the plan. */ + public ExplainPlanAssert rewriteCount(int expected) { + List rewrites = attributes.getRewrites(); + int actual = rewrites == null ? 0 : rewrites.size(); + assertEquals(at("rewrites.size"), expected, actual); + return this; + } + + /** Assert the i-th rewrite breadcrumb. */ + public ExplainPlanAssert rewrite(int i, String expected) { + List rewrites = attributes.getRewrites(); + assertNotNull(at("rewrites") + " must not be null", rewrites); + assertTrue(at("rewrites") + " has no index " + i + " (size=" + rewrites.size() + ")", + i >= 0 && i < rewrites.size()); + assertEquals(at("rewrites[" + i + "]"), expected, rewrites.get(i)); + return this; + } + + /** Assert the entire ordered rewrite breadcrumb list matches {@code expected}. */ + public ExplainPlanAssert rewrites(String... expected) { + List actual = attributes.getRewrites(); + List actualOrEmpty = actual == null ? Collections. emptyList() : actual; + assertEquals(at("rewrites"), Arrays.asList(expected), actualOrEmpty); + return this; + } + + /** Assert the rewrite breadcrumb list contains {@code expected}. */ + public ExplainPlanAssert rewriteContains(String expected) { + List rewrites = attributes.getRewrites(); + assertNotNull(at("rewrites") + " must not be null", rewrites); + assertTrue(at("rewrites") + " expected to contain '" + expected + "' but was " + rewrites, + rewrites.contains(expected)); + return this; + } + + /** Assert that no rewrite breadcrumbs were disclosed (null or empty). */ + public ExplainPlanAssert rewritesNone() { + List rewrites = attributes.getRewrites(); + assertTrue(at("rewrites") + " expected none but was " + rewrites, + rewrites == null || rewrites.isEmpty()); + return this; + } + + /** + * Return to the parent assertion after navigating into {@link #rhs()} or {@link #subPlan(int)}. + */ + public ExplainPlanAssert end() { + assertNotNull("end() called on a root ExplainPlanAssert", parent); + return parent; + } + + private String at(String field) { + return context + "." + field; + } + + private static String trim(String s) { + return s == null ? null : s.trim(); + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainTextNormalizer.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainTextNormalizer.java new file mode 100644 index 00000000000..c8e1d1ef460 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainTextNormalizer.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.query.explain; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +/** + * Elides cluster- and connection-specific details from the {@code List} returned by + * {@code ExplainPlan.getPlanSteps()} so the EXPLAIN text can be compared across environments. + */ +public final class ExplainTextNormalizer { + + // Matches any non-negative integer immediately before -CHUNK. + private static final Pattern CHUNK_COUNT = Pattern.compile("\\b\\d+-CHUNK\\b"); + + // Matches the iterator parallelism count. + private static final Pattern WAY_COUNT = Pattern.compile("\\b\\d+-WAY\\b"); + + // Matches the stats row and byte counts. + private static final Pattern ROWS_BYTES = Pattern.compile("\\d+ ROWS \\d+ BYTES\\s*"); + + // Matches the planned regions count on the REGIONS PLANNED line. + private static final Pattern REGIONS_PLANNED = Pattern.compile("REGIONS PLANNED \\d+"); + + // Matches the region locations line. + private static final String REGION_LOCATIONS_PREFIX = " (region locations = "; + + /** + * @param raw the result of {@code ExplainPlan.getPlanSteps()} + * @return a new list with cluster/connection-specific detail elided. The original list is not + * mutated. + */ + public List normalize(List raw) { + // Temp alias state is shared across every line of a single plan so that an alias appearing in + // both a sub-plan body and a parent level dynamic filter renumbers consistently. + TempAliasRenumberer aliases = new TempAliasRenumberer(); + List out = new ArrayList<>(raw.size()); + for (String line : raw) { + if (line == null) { + out.add(null); + continue; + } + // Drop region-location lines outright + if (line.contains(REGION_LOCATIONS_PREFIX)) { + continue; + } + String normalized = line; + normalized = CHUNK_COUNT.matcher(normalized).replaceAll("-CHUNK"); + normalized = WAY_COUNT.matcher(normalized).replaceAll("-WAY"); + normalized = ROWS_BYTES.matcher(normalized).replaceAll(""); + normalized = REGIONS_PLANNED.matcher(normalized).replaceAll("REGIONS PLANNED "); + normalized = aliases.rewrite(normalized); + out.add(normalized); + } + return out; + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/TempAliasRenumberer.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/TempAliasRenumberer.java new file mode 100644 index 00000000000..464b986d914 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/TempAliasRenumberer.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.query.explain; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Renumbers Phoenix temp-alias tokens of the form {@code $}. + *

+ * Temp aliases are produced by {@code ParseNodeFactory.createTempAlias()} from a JVM-global + * {@code AtomicInteger}, so the literal numbers in an EXPLAIN line depend on what was compiled + * earlier in the same JVM. + *

+ * The first distinct alias encountered becomes {@code $1}, the second {@code $2}, and so on, so + * that structural relationships are preserved within one plan. + */ +final class TempAliasRenumberer { + + private static final Pattern TEMP_ALIAS = Pattern.compile("\\$\\d+"); + + private final Map mapping = new HashMap<>(); + private int next = 0; + + /** + * @param s arbitrary text + * @return {@code s} with each {@code $} token replaced by its renumbered alias, or + * {@code s} unchanged when no token appears. {@code null} input returns {@code null}. + */ + String rewrite(String s) { + if (s == null) { + return null; + } + Matcher m = TEMP_ALIAS.matcher(s); + if (!m.find()) { + return s; + } + m.reset(); + StringBuffer sb = new StringBuffer(); + while (m.find()) { + String tok = m.group(); + String repl = mapping.get(tok); + if (repl == null) { + next++; + repl = "$" + next; + mapping.put(tok, repl); + } + m.appendReplacement(sb, Matcher.quoteReplacement(repl)); + } + m.appendTail(sb); + return sb.toString(); + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index 69a191b5c2d..2b7a37ddf09 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -1445,15 +1445,18 @@ public static boolean hasFilter(Scan scan, Class filterClass) public static JoinTable getJoinTable(String query, PhoenixConnection connection) throws SQLException { SQLParser parser = new SQLParser(query); - SelectStatement select = SubselectRewriter.flatten(parser.parseQuery(), connection); + PhoenixStatement stmt = connection.createStatement().unwrap(PhoenixStatement.class); + StatementContext rewriteContext = StatementContext.forRewrite(stmt); + SelectStatement select = + SubselectRewriter.flatten(parser.parseQuery(), connection, rewriteContext); ColumnResolver resolver = FromCompiler.getResolverForQuery(select, connection); select = StatementNormalizer.normalize(select, resolver); - SelectStatement transformedSelect = SubqueryRewriter.transform(select, resolver, connection); + SelectStatement transformedSelect = + SubqueryRewriter.transform(select, resolver, connection, rewriteContext); if (transformedSelect != select) { resolver = FromCompiler.getResolverForQuery(transformedSelect, connection); select = StatementNormalizer.normalize(transformedSelect, resolver); } - PhoenixStatement stmt = connection.createStatement().unwrap(PhoenixStatement.class); return JoinCompiler.compile(stmt, select, resolver); }