diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/Bindings.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/Bindings.java index 38732a7711..c0ebd0222f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/Bindings.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/Bindings.java @@ -22,15 +22,20 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.planprotos.PParameterComparison.PBindingKind; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.util.pair.Pair; +import com.google.common.base.Suppliers; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; /** * A map of bound parameter values passed to query evaluation. @@ -117,9 +122,13 @@ public static Internal fromProto(@Nonnull final PlanSerializationContext seriali public static final Bindings EMPTY_BINDINGS = new Bindings(); + @Nonnull + private Supplier> boundCorrelationAliasesSupplier; + private Bindings(@Nullable Bindings parent) { this.values = new HashMap<>(); this.parent = parent; + this.boundCorrelationAliasesSupplier = Suppliers.memoize(this::computeBoundCorrelationAliases); } public Bindings() { @@ -165,6 +174,20 @@ public List> asMappingList() { return resultBuilder.build(); } + @Nonnull + public Set getBoundCorrelationAliases() { + return boundCorrelationAliasesSupplier.get(); + } + + @Nonnull + private Set computeBoundCorrelationAliases() { + return asMappingList() + .stream() + .filter(entry -> Bindings.Internal.CORRELATION.isOfType(entry.getKey())) + .map(entry -> CorrelationIdentifier.of(Bindings.Internal.CORRELATION.identifier(entry.getKey()))) + .collect(ImmutableSet.toImmutableSet()); + } + @Override public String toString() { return "Bindings(" + asMappingList() + ")"; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java index 53a904c256..6844696248 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java @@ -405,6 +405,11 @@ private void planPartial(@Nonnull final Supplier referenceSupplier, @Nonnull final Function contextCreatorFunction, @Nonnull final EvaluationContext evaluationContext) { this.currentRoot = referenceSupplier.get(); + + // run sanity check to make sure that all aliases handed in can be uniquely resolved + Debugger.sanityCheck(() -> + currentRoot.verifyCorrelationsRecursive(evaluationContext.getBindings().getBoundCorrelationAliases())); + this.planContext = contextCreatorFunction.apply(currentRoot); this.evaluationContext = evaluationContext; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesRuleCall.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesRuleCall.java index 3a342ad042..0dc949eacf 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesRuleCall.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesRuleCall.java @@ -215,7 +215,7 @@ public void yieldUnknownExpression(@Nonnull final RelationalExpression expressio } private void yieldExpression(@Nonnull final RelationalExpression expression, final boolean isFinal) { - verifyChildrenMemoized(expression); + validateNewExpression(expression); if (isFinal) { if (root.insertFinalExpression(expression)) { newFinalExpressions.add(expression); @@ -229,6 +229,12 @@ private void yieldExpression(@Nonnull final RelationalExpression expression, fin } } + protected void validateNewExpression(@Nonnull final RelationalExpression expression) { + Debugger.sanityCheck(() -> verifyChildrenMemoized(expression)); + Debugger.sanityCheck(() -> getRoot().verifyCorrelationsForNewExpression(traversal, expression, + getEvaluationContext())); + } + private void verifyChildrenMemoized(@Nonnull RelationalExpression expression) { for (final var quantifier : expression.getQuantifiers()) { final var rangesOver = quantifier.getRangesOver(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Reference.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Reference.java index c89fd150e3..0a805ba817 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Reference.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Reference.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.query.plan.cascades; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.query.plan.HeuristicPlanner; import com.apple.foundationdb.record.query.plan.cascades.debug.Debugger; @@ -40,6 +41,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.SetMultimap; +import com.google.common.collect.Sets; import com.google.errorprone.annotations.CanIgnoreReturnValue; import javax.annotation.Nonnull; @@ -708,6 +710,132 @@ public boolean addPartialMatchForCandidate(final MatchCandidate candidate, final return partialMatchMap.put(candidate, partialMatch); } + public void verifyCorrelationsForNewExpression(@Nonnull final Traversal traversal, + @Nonnull final RelationalExpression expression, + @Nonnull final EvaluationContext evaluationContext) { + final Set correlatedToWithoutChildren; + if (expression instanceof RelationalExpressionWithChildren) { + correlatedToWithoutChildren = ((RelationalExpressionWithChildren)expression).getCorrelatedToWithoutChildren(); + } else { + correlatedToWithoutChildren = expression.getCorrelatedTo(); + } + + final var visibleThroughEvaluationContext = evaluationContext.getBindings().getBoundCorrelationAliases(); + final var locallyVisibleAliases = expression.getLocallyVisibleAliases(); + + final var currentResolvedCorrelatedToBuilder = + ImmutableSet.builder(); + final var currentUnresolvedCorrelatedToBuilder = + ImmutableSet.builder(); + for (final var unresolvedAlias : correlatedToWithoutChildren) { + if (visibleThroughEvaluationContext.contains(unresolvedAlias) || + locallyVisibleAliases.contains(unresolvedAlias)) { + currentResolvedCorrelatedToBuilder.add(unresolvedAlias); + } else { + // still unresolved + currentUnresolvedCorrelatedToBuilder.add(unresolvedAlias); + } + } + + final var currentUnresolvedCorrelatedTo = + currentUnresolvedCorrelatedToBuilder.build(); + final var currentResolvedCorrelatedTo = + currentResolvedCorrelatedToBuilder.build(); + + final var parentRefPaths = traversal.getParentRefPaths(this); + + if (parentRefPaths.isEmpty()) { + Verify.verify(currentUnresolvedCorrelatedTo.isEmpty(), "unresolved aliases: " + currentUnresolvedCorrelatedTo); + } else { + for (final var parentRefPath : parentRefPaths) { + final var parentReference = parentRefPath.getReference(); + parentReference.verifyCorrelationsForNewExpressionRecursive(traversal, parentRefPath.getExpression(), + parentRefPath.getQuantifier(), currentUnresolvedCorrelatedTo, currentResolvedCorrelatedTo); + } + } + } + + private void verifyCorrelationsForNewExpressionRecursive(@Nonnull final Traversal traversal, + @Nonnull final RelationalExpression expression, + @Nonnull final Quantifier quantifier, + @Nonnull final Set unresolvedCorrelatedTo, + @Nonnull final Set resolvedCorrelatedTo) { + + final Set localVisibleAliases; + if (expression.canCorrelate()) { + final var allLocallyVisibleAliases = expression.getLocallyVisibleAliases(); + Verify.verify(allLocallyVisibleAliases.contains(quantifier.getAlias())); + localVisibleAliases = + Sets.difference(allLocallyVisibleAliases, ImmutableSet.of(quantifier.getAlias())); + } else { + localVisibleAliases = ImmutableSet.of(); + } + + final var intersection = Sets.intersection(localVisibleAliases, resolvedCorrelatedTo); + Verify.verify(intersection.isEmpty(), "ambiguous aliases: " + intersection); + final var currentResolvedCorrelatedToBuilder = + ImmutableSet.builder(); + currentResolvedCorrelatedToBuilder.addAll(resolvedCorrelatedTo); + + final var currentUnresolvedCorrelatedToBuilder = + ImmutableSet.builder(); + for (final var unresolvedAlias : unresolvedCorrelatedTo) { + if (localVisibleAliases.contains(unresolvedAlias)) { + currentResolvedCorrelatedToBuilder.add(unresolvedAlias); + } else { + // still unresolved + currentUnresolvedCorrelatedToBuilder.add(unresolvedAlias); + } + } + + final var currentUnresolvedCorrelatedTo = + currentUnresolvedCorrelatedToBuilder.build(); + final var currentResolvedCorrelatedTo = + currentResolvedCorrelatedToBuilder.build(); + + final var parentRefPaths = traversal.getParentRefPaths(this); + + if (parentRefPaths.isEmpty()) { + Verify.verify(currentUnresolvedCorrelatedTo.isEmpty(), "unresolved aliases: " + + currentUnresolvedCorrelatedTo); + } else { + for (final var parentRefPath : parentRefPaths) { + final var parentReference = parentRefPath.getReference(); + parentReference.verifyCorrelationsForNewExpressionRecursive(traversal, + parentRefPath.getExpression(), parentRefPath.getQuantifier(), currentUnresolvedCorrelatedTo, + currentResolvedCorrelatedTo); + } + } + } + + public void verifyCorrelationsRecursive(@Nonnull final Set visibleAliases) { + for (final var expression : getAllMemberExpressions()) { + final var locallyVisibleAliases = expression.getLocallyVisibleAliases(); + final var intersection = Sets.intersection(visibleAliases, locallyVisibleAliases); + Verify.verify(intersection.isEmpty(), "ambiguous aliases: ", intersection); + + final var allVisibleAliases = Sets.union(visibleAliases, locallyVisibleAliases); + final Set correlatedToWithoutChildren; + if (expression instanceof RelationalExpressionWithChildren) { + correlatedToWithoutChildren = ((RelationalExpressionWithChildren)expression).getCorrelatedToWithoutChildren(); + } else { + correlatedToWithoutChildren = expression.getCorrelatedTo(); + } + final var difference = Sets.difference(correlatedToWithoutChildren, allVisibleAliases); + Verify.verify(difference.isEmpty(), "unresolved aliases: " + difference); + + for (final var quantifier : expression.getQuantifiers()) { + final var lowerReference = quantifier.getRangesOver(); + if (expression.canCorrelate()) { + lowerReference.verifyCorrelationsRecursive(Sets.difference(allVisibleAliases, + ImmutableSet.of(quantifier.getAlias()))); + } else { + lowerReference.verifyCorrelationsRecursive(visibleAliases); + } + } + } + } + /** * Method to render the graph rooted at this reference. This is needed for graph integration into IntelliJ as * IntelliJ only ever evaluates selfish methods. Add this method as a custom renderer for the type diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RecursiveUnionExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RecursiveUnionExpression.java index e48a7cc116..23ca8d8428 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RecursiveUnionExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RecursiveUnionExpression.java @@ -86,6 +86,13 @@ public boolean canCorrelate() { return true; } + @Nonnull + @Override + public Set getLocallyVisibleAliases() { + return ImmutableSet.of(tempTableInsertAlias, tempTableScanAlias, initialStateQuantifier.getAlias(), + recursiveStateQuantifier.getAlias()); + } + @Nonnull @Override public Value getResultValue() { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RelationalExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RelationalExpression.java index d540aee3c6..7022a04aa3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RelationalExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/RelationalExpression.java @@ -260,6 +260,11 @@ default PartiallyOrderedSet getCorrelationOrder() { return PartiallyOrderedSet.empty(); } + @Nonnull + default Set getLocallyVisibleAliases() { + return Quantifiers.aliases(getQuantifiers()); + } + boolean equalsWithoutChildren(@Nonnull RelationalExpression other, @Nonnull AliasMap equivalences); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementInJoinRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementInJoinRule.java index 5f66e07b62..8819e78fc5 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementInJoinRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementInJoinRule.java @@ -45,6 +45,7 @@ import com.apple.foundationdb.record.query.plan.cascades.matching.structure.CollectionMatcher; import com.apple.foundationdb.record.query.plan.cascades.properties.OrderingProperty; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ParameterValue; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.plans.InComparandSource; @@ -383,6 +384,7 @@ private static Stream> suffixForRemainingExplodes(@ private static boolean isSupportedExplodeValue(@Nonnull final Value explodeValue) { return explodeValue instanceof LiteralValue || explodeValue instanceof QuantifiedObjectValue || + explodeValue instanceof ParameterValue || explodeValue.isConstant(); } @@ -417,6 +419,11 @@ private static InSource computeInSource(@Nonnull final Value explodeValue, return attemptedSortOrder == null ? new InParameterSource(bindingName, alias) : new SortedInParameterSource(bindingName, alias, attemptedSortOrder.isAnyDescending()); // TODO needs to distinguish between different descending orders + } else if (explodeValue instanceof ParameterValue) { + final var alias = ((ParameterValue)explodeValue).getBindingName(); + return attemptedSortOrder == null + ? new InParameterSource(bindingName, alias) + : new SortedInParameterSource(bindingName, alias, attemptedSortOrder.isAnyDescending()); // TODO needs to distinguish between different descending orders } else if (explodeValue.isConstant()) { return attemptedSortOrder == null ? new InComparandSource(bindingName, new Comparisons.ValueComparison(Comparisons.Type.IN, explodeValue)) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementInUnionRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementInUnionRule.java index f7d48a1a6a..1afc04dded 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementInUnionRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementInUnionRule.java @@ -40,6 +40,7 @@ import com.apple.foundationdb.record.query.plan.cascades.matching.structure.CollectionMatcher; import com.apple.foundationdb.record.query.plan.cascades.properties.OrderingProperty; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ParameterValue; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.plans.InComparandSource; @@ -146,6 +147,9 @@ public void onMatch(@Nonnull final ImplementationCascadesRuleCall call) { } else if (explodeCollectionValue instanceof QuantifiedObjectValue) { inSource = new InParameterSource(bindingName, ((QuantifiedObjectValue)explodeCollectionValue).getAlias().getId()); + } else if (explodeCollectionValue instanceof ParameterValue) { + inSource = new InParameterSource(bindingName, + ((ParameterValue)explodeCollectionValue).getBindingName()); } else if (explodeCollectionValue.isConstant()) { inSource = new InComparandSource(bindingName, new Comparisons.ValueComparison(Comparisons.Type.IN, explodeCollectionValue)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/InComparisonToExplodeRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/InComparisonToExplodeRule.java index 44f6c2fb9a..d559bf9dd6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/InComparisonToExplodeRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/InComparisonToExplodeRule.java @@ -24,9 +24,8 @@ import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.cascades.Column; -import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; -import com.apple.foundationdb.record.query.plan.cascades.ExplorationCascadesRuleCall; import com.apple.foundationdb.record.query.plan.cascades.ExplorationCascadesRule; +import com.apple.foundationdb.record.query.plan.cascades.ExplorationCascadesRuleCall; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.expressions.ExplodeExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression; @@ -39,6 +38,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.BooleanValue; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ParameterValue; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.RelOpValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; @@ -178,7 +178,7 @@ public void onMatch(@Nonnull final ExplorationCascadesRuleCall call) { new Comparisons.ValueComparison(Comparisons.Type.EQUALS, QuantifiedObjectValue.of(newQuantifier.getAlias(), elementType)))); } else if (comparison instanceof Comparisons.ParameterComparison) { - explodeExpression = new ExplodeExpression(QuantifiedObjectValue.of(CorrelationIdentifier.of(((Comparisons.ParameterComparison)comparison).getParameter()), new Type.Array(elementType))); + explodeExpression = new ExplodeExpression(ParameterValue.of(((Comparisons.ParameterComparison)comparison).getParameter(), new Type.Array(elementType))); newQuantifier = Quantifier.forEach(call.memoizeExploratoryExpression(explodeExpression)); transformedPredicates.add(new ValuePredicate(value, new Comparisons.ValueComparison(Comparisons.Type.EQUALS, QuantifiedObjectValue.of(newQuantifier.getAlias(), elementType)))); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ParameterValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ParameterValue.java new file mode 100644 index 0000000000..3ecc00971a --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ParameterValue.java @@ -0,0 +1,190 @@ +/* + * QuantifiedObjectValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PParameterValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.ConstrainedBoolean; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * A value representing the quantifier as an object. For example, this is used to represent non-nested repeated fields. + */ +@API(API.Status.EXPERIMENTAL) +public class ParameterValue extends AbstractValue implements LeafValue { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Parameter-Value"); + + @Nonnull + private final String bindingName; + @Nonnull + private final Type resultType; + + private ParameterValue(@Nonnull final String bindingName, @Nonnull final Type resultType) { + this.bindingName = bindingName; + this.resultType = resultType; + } + + @Nonnull + public String getBindingName() { + return bindingName; + } + + @Nonnull + @Override + public Type getResultType() { + return resultType; + } + + @Nonnull + @Override + public Value rebaseLeaf(@Nonnull final CorrelationIdentifier targetAlias) { + return this; + } + + @Nonnull + @Override + public Value replaceReferenceWithField(@Nonnull final FieldValue fieldValue) { + return fieldValue; + } + + @Nullable + @Override + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + return context.getBinding(bindingName); + } + + @Nonnull + @Override + protected Iterable computeChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH); + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + return PlanHashable.objectsPlanHash(mode, BASE_HASH); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { + Verify.verify(Iterables.isEmpty(explainSuppliers)); + return ExplainTokensWithPrecedence.of(new ExplainTokens().addIdentifier(bindingName)); + } + + @Override + public int hashCode() { + return semanticHashCode(); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @SpotBugsSuppressWarnings("EQ_UNUSUAL") + @Override + public boolean equals(final Object other) { + return semanticEquals(other, AliasMap.emptyMap()); + } + + @Nonnull + @Override + public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { + return super.equalsWithoutChildren(other) + .filter(ignored -> bindingName.equals(((ParameterValue)other).getBindingName())); + } + + @Nonnull + @Override + public Value with(@Nonnull final Type type) { + return ParameterValue.of(bindingName, type); + } + + @Nonnull + @Override + public PParameterValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + PParameterValue.Builder builder = PParameterValue.newBuilder(); + builder.setBindingName(bindingName); + builder.setResultType(resultType.toTypeProto(serializationContext)); + return builder.build(); + } + + @Nonnull + @Override + public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { + final var specificValueProto = toProto(serializationContext); + return PValue.newBuilder().setParameterValue(specificValueProto).build(); + } + + @Nonnull + public static ParameterValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PParameterValue parameterValueProto) { + return new ParameterValue(Objects.requireNonNull(parameterValueProto.getBindingName()), + Type.fromTypeProto(serializationContext, Objects.requireNonNull(parameterValueProto.getResultType()))); + } + + @Nonnull + public static ParameterValue of(@Nonnull final String bindingName, @Nonnull final Type resultType) { + return new ParameterValue(bindingName, resultType); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PParameterValue.class; + } + + @Nonnull + @Override + public ParameterValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PParameterValue parameterValueProto) { + return ParameterValue.fromProto(serializationContext, parameterValueProto); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicatesFilterPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicatesFilterPlan.java index 71fd1d25a4..d611ac0736 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicatesFilterPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicatesFilterPlan.java @@ -157,7 +157,7 @@ public RecordQueryPredicatesFilterPlan translateCorrelations(@Nonnull final Tran @Nonnull @Override public RecordQueryPlanWithChild withChild(@Nonnull final Reference childRef) { - return new RecordQueryPredicatesFilterPlan(Quantifier.physical(childRef), getPredicates()); + return new RecordQueryPredicatesFilterPlan(getInner().overNewReference(childRef), getPredicates()); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryRecursiveUnionPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryRecursiveUnionPlan.java index 51e8d69577..36a96ccc6d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryRecursiveUnionPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryRecursiveUnionPlan.java @@ -143,6 +143,18 @@ public Set getCorrelatedToWithoutChildren() { return ImmutableSet.of(); } + @Override + public boolean canCorrelate() { + return true; + } + + @Nonnull + @Override + public Set getLocallyVisibleAliases() { + return ImmutableSet.of(tempTableInsertAlias, tempTableScanAlias, initialStateQuantifier.getAlias(), + recursiveStateQuantifier.getAlias()); + } + @Nonnull @Override public RecordCursor executePlan(@Nonnull final FDBRecordStoreBase store, diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 9a1c56337c..061a240c0c 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -253,6 +253,7 @@ message PValue { PMacroFunctionValue macro_function_value = 47; PRangeValue range_value = 48; PFirstOrDefaultStreamingValue first_or_default_streaming_value = 49; + PParameterValue parameter_value = 50; } } @@ -473,6 +474,11 @@ message PFromOrderedBytesValue { optional PType result_type = 3; } +message PParameterValue { + optional string bindingName = 1; + optional PType result_type = 2; +} + enum PDirection { ASC_NULLS_FIRST = 1; ASC_NULLS_LAST = 2; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java index 2ca83d6c6b..d8eae11a0b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java @@ -25,11 +25,10 @@ import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; -import com.apple.test.Tags; +import com.apple.test.SuperSlow; import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -48,7 +47,7 @@ /** * Additional tests for Multidimensional Index around concurrency and large data-sets. */ -@Tag(Tags.Slow) +@SuperSlow class SlowMultidimensionalIndexTest extends MultidimensionalIndexTestBase { static Stream argumentsForBasicReads() { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/RecursiveUnionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/RecursiveUnionTest.java index e893923fe7..01fd9601d2 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/RecursiveUnionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/RecursiveUnionTest.java @@ -36,6 +36,7 @@ import com.apple.foundationdb.record.query.plan.cascades.GraphExpansion; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.explain.ExplainPlanVisitor; import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression; @@ -48,7 +49,6 @@ import com.apple.foundationdb.record.query.plan.cascades.values.ArithmeticValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; -import com.apple.foundationdb.record.query.plan.cascades.explain.ExplainPlanVisitor; import com.apple.foundationdb.record.query.plan.plans.QueryResult; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; @@ -321,9 +321,9 @@ private List multiplesOf(@Nonnull final List initial, long limit) th final var logicalPlan = Reference.initialOf(LogicalSortExpression.unsorted(Quantifier.forEach(Reference.initialOf(recursiveUnionPlan)))); final var cascadesPlanner = (CascadesPlanner)planner; - final var plan = cascadesPlanner.planGraph(() -> logicalPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, EvaluationContext.empty()).getPlan(); + final var evaluationContext = putTempTableInContext(seedingTempTableAlias, seedingTempTable, null); + final var plan = cascadesPlanner.planGraph(() -> logicalPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, evaluationContext).getPlan(); - var evaluationContext = putTempTableInContext(seedingTempTableAlias, seedingTempTable, null); return extractResultsAsIds(context, plan, evaluationContext).stream().collect(ImmutableList.toImmutableList()); } } @@ -485,11 +485,15 @@ private HierarchyExecutionResult hierarchicalQuery(@Nonnull final Map seedingTempTable.add(queryResult(id, parent))); var evaluationContext = setUpPlanContext(plan, seedingTempTableAlias, seedingTempTable); @@ -539,7 +543,8 @@ private HierarchyExecutionResult executeHierarchyPlan(@Nonnull final RecordQuery private RecordQueryPlan createAndOptimizeHierarchyQuery(@Nonnull final CorrelationIdentifier seedingTempTableAlias, @Nonnull final CorrelationIdentifier insertTempTableAlias, @Nonnull final CorrelationIdentifier scanTempTableAlias, - @Nonnull final BiFunction queryPredicate) { + @Nonnull final BiFunction queryPredicate, + @Nonnull final EvaluationContext evaluationContext) { final var ttScanSeeding = Quantifier.forEach(Reference.initialOf(TempTableScanExpression.ofCorrelated(seedingTempTableAlias, getHierarchyType()))); var selectExpression = GraphExpansion.builder() .addAllResultColumns(ImmutableList.of(getIdCol(ttScanSeeding), getParentCol(ttScanSeeding))).addQuantifier(ttScanSeeding) @@ -569,7 +574,7 @@ private RecordQueryPlan createAndOptimizeHierarchyQuery(@Nonnull final Correlati final var logicalPlan = Reference.initialOf(LogicalSortExpression.unsorted(Quantifier.forEach(Reference.initialOf(recursiveUnionPlan)))); final var cascadesPlanner = (CascadesPlanner)planner; - return cascadesPlanner.planGraph(() -> logicalPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, EvaluationContext.empty()).getPlan(); + return cascadesPlanner.planGraph(() -> logicalPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, evaluationContext).getPlan(); } @Nonnull diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTest.java index 1169baff2d..3f3090f2a2 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTest.java @@ -71,9 +71,9 @@ public class TempTableTest extends TempTableTestBase { void scanTempTableWorksCorrectly() throws Exception { try (FDBRecordContext context = openContext()) { // select id, value from . - final var tempTable = tempTableInstance(); final var tempTableId = CorrelationIdentifier.uniqueID(); - final var plan = createAndOptimizeTempTableScanPlan(tempTableId); + final var tempTable = tempTableInstance(); + final var plan = createAndOptimizeTempTableScanPlan(tempTableId, tempTable); addSampleDataToTempTable(tempTable); final var expectedResults = ImmutableList.of(Pair.of(42L, "fortySecondValue"), Pair.of(45L, "fortyFifthValue")); @@ -95,7 +95,9 @@ void scanTempTableWithPredicateWorksCorrectly() throws Exception { .addQuantifier(tempTableScanQun); final var logicalPlan = Reference.initialOf(LogicalSortExpression.unsorted(Quantifier.forEach(Reference.initialOf(selectExpressionBuilder.build().buildSelect())))); final var cascadesPlanner = (CascadesPlanner)planner; - final var plan = cascadesPlanner.planGraph(() -> logicalPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, EvaluationContext.empty()).getPlan(); + final var compilationEvaluationContext = putTempTableInContext(tempTableId, tempTable, EvaluationContext.empty()); + final var plan = cascadesPlanner.planGraph(() -> logicalPlan, + Optional.empty(), IndexQueryabilityFilter.TRUE, compilationEvaluationContext).getPlan(); assertMatchesExactly(plan, mapPlan(predicatesFilterPlan(tempTableScanPlan()).where(predicates(only(valuePredicate(fieldValueWithFieldNames("id"), new Comparisons.SimpleComparison(Comparisons.Type.LESS_THAN, 44L))))))); assertEquals(ImmutableList.of(Pair.of(42L, "fortySecondValue")), collectResults(context, plan, tempTable, tempTableId)); @@ -118,14 +120,15 @@ void insertIntoTempTableWorksCorrectly() throws Exception { final var insertPlan = Reference.initialOf(LogicalSortExpression.unsorted(qun)); final var cascadesPlanner = (CascadesPlanner)planner; + final var compilationEvaluationContext = putTempTableInContext(tempTableId, tempTable, EvaluationContext.empty()); var plan = cascadesPlanner.planGraph(() -> insertPlan, Optional.empty(), - IndexQueryabilityFilter.TRUE, EvaluationContext.empty()).getPlan(); + IndexQueryabilityFilter.TRUE, compilationEvaluationContext).getPlan(); assertMatchesExactly(plan, tempTableInsertPlan(explodePlan())); final var evaluationContext = putTempTableInContext(tempTableId, tempTable, null); fetchResultValues(context, plan, Function.identity(), evaluationContext, c -> { }, ExecuteProperties.SERIAL_EXECUTE); // select id, value from tq1 | tq1 is a temporary table. - plan = createAndOptimizeTempTableScanPlan(tempTableId); + plan = createAndOptimizeTempTableScanPlan(tempTableId, tempTable); assertEquals(ImmutableList.of(Pair.of(1L, "first"), Pair.of(2L, "second")), collectResults(context, plan, tempTable, tempTableId)); } @@ -149,8 +152,9 @@ void insertIntoTempTableWorksCorrectlyAcrossContinuations() throws Exception { final var insertPlan = Reference.initialOf(LogicalSortExpression.unsorted(qun)); final var cascadesPlanner = (CascadesPlanner)planner; + final var compilationEvaluationContext = putTempTableInContext(tempTableId, tempTable, EvaluationContext.empty()); planToResume = cascadesPlanner.planGraph(() -> insertPlan, Optional.empty(), - IndexQueryabilityFilter.TRUE, EvaluationContext.empty()).getPlan(); + IndexQueryabilityFilter.TRUE, compilationEvaluationContext).getPlan(); assertMatchesExactly(planToResume, tempTableInsertPlan(explodePlan())); final var evaluationContext = setUpPlanContext(planToResume, tempTableId, tempTable); try (RecordCursorIterator cursor = planToResume.executePlan(recordStore, evaluationContext, @@ -190,7 +194,7 @@ void scanTempTableWithPredicateWorksCorrectlyAcrossContinuations() { tempTable.add(queryResult(2L, "two")); tempTable.add(queryResult(3L, "three")); tempTable.add(queryResult(4L, "four")); - planToResume = createAndOptimizeTempTableScanPlan(tempTableId); + planToResume = createAndOptimizeTempTableScanPlan(tempTableId, tempTable); final var evaluationContext = setUpPlanContext(planToResume, tempTableId, tempTable); // Read the first two elements "one", "two". diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTestBase.java index a29248bfb5..006aa4eeb0 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/TempTableTestBase.java @@ -254,15 +254,17 @@ static RecordConstructorValue rcv(long id, long parent, @Nonnull final String va } @Nonnull - RecordQueryPlan createAndOptimizeTempTableScanPlan(@Nonnull final CorrelationIdentifier tempTableId) { + RecordQueryPlan createAndOptimizeTempTableScanPlan(@Nonnull final CorrelationIdentifier tempTableId, + @Nonnull final TempTable tempTable) { final var tempTableScanQun = Quantifier.forEach(Reference.initialOf(TempTableScanExpression.ofCorrelated(tempTableId, getTempTableType()))); final var selectExpressionBuilder = GraphExpansion.builder() .addAllResultColumns(ImmutableList.of(getIdCol(tempTableScanQun), getValueCol(tempTableScanQun))) .addQuantifier(tempTableScanQun); final var logicalPlan = Reference.initialOf(LogicalSortExpression.unsorted(Quantifier.forEach(Reference.initialOf(selectExpressionBuilder.build().buildSelect())))); final var cascadesPlanner = (CascadesPlanner)planner; + final var compileTimeEvaluationContext = putTempTableInContext(tempTableId, tempTable, EvaluationContext.empty()); final var plan = cascadesPlanner.planGraph(() -> logicalPlan, Optional.empty(), IndexQueryabilityFilter.TRUE, - EvaluationContext.empty()).getPlan(); + compileTimeEvaluationContext).getPlan(); assertMatchesExactly(plan, mapPlan(tempTableScanPlan())); return plan; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/RuleTestHelper.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/RuleTestHelper.java index 846162824d..c610a912e7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/RuleTestHelper.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/RuleTestHelper.java @@ -36,6 +36,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.RangeValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.cascades.values.translation.ToUniqueAliasesTranslationMap; +import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -144,13 +145,23 @@ public RuleTestHelper(@Nonnull CascadesRule rule } @Nonnull - private TestRuleExecution run(RelationalExpression original) { - // copy the graph handed in so the caller can modify it at will afterwards and won't see the effects of + @SuppressWarnings("PMD.CompareObjectsWithEquals") + + private TestRuleExecution run(Reference root, Reference group, RelationalExpression original) { + // copy the graph handed in so the caller can modify it at will afterward and won't see the effects of // rewriting and planning - final var copiedOriginal = - Iterables.getOnlyElement(References.rebaseGraphs(ImmutableList.of(Reference.initialOf(original)), - Memoizer.noMemoization(PlannerStage.INITIAL), new ToUniqueAliasesTranslationMap(), false)).get(); - ensureStage(PlannerStage.CANONICAL, copiedOriginal); + final var copiedOriginals = + References.rebaseGraphs(ImmutableList.of(root, group), + Memoizer.noMemoization(PlannerStage.INITIAL), new ToUniqueAliasesTranslationMap(), false); + + Verify.verify(copiedOriginals.size() == 2); + final var copiedRoot = copiedOriginals.get(0); + final var copiedGroup = copiedOriginals.get(1); + final var copiedExpression = copiedGroup.get(); + + ensureStage(PlannerStage.CANONICAL, copiedRoot); + + final var traversal = Traversal.withRoot(copiedRoot); if (rule instanceof ImplementationCascadesRule) { // // Descend the copied original to: @@ -162,35 +173,37 @@ private TestRuleExecution run(RelationalExpression original) { // here for exploration rules as they attempt to share sub graphs which would then make the verification // of the result of the rule difficult. // - preExploreForRule(copiedOriginal, false); + preExploreForRule(copiedExpression, traversal, false); } - Reference ref = Reference.ofExploratoryExpression(PlannerStage.CANONICAL, copiedOriginal); PlanContext planContext = new FakePlanContext(); - return TestRuleExecution.applyRule(planContext, rule, ref, EvaluationContext.EMPTY); + return TestRuleExecution.applyRule(planContext, rule, copiedGroup, true, + traversal, EvaluationContext.empty()); } - public void ensureStage(@Nonnull PlannerStage plannerStage, @Nonnull RelationalExpression expression) { - for (Quantifier qun : expression.getQuantifiers()) { - Reference ref = qun.getRangesOver(); - if (ref.getPlannerStage() != plannerStage) { - ref.advancePlannerStageUnchecked(plannerStage); - } - for (RelationalExpression refMember : ref.getAllMemberExpressions()) { - ensureStage(plannerStage, refMember); + public void ensureStage(@Nonnull PlannerStage plannerStage, @Nonnull Reference reference) { + if (reference.getPlannerStage() != plannerStage) { + reference.advancePlannerStageUnchecked(plannerStage); + } + for (RelationalExpression expression : reference.getAllMemberExpressions()) { + for (Quantifier qun : expression.getQuantifiers()) { + Reference childReference = qun.getRangesOver(); + ensureStage(plannerStage, childReference); } } } public void preExploreForRule(@Nonnull final RelationalExpression expression, + @Nonnull final Traversal traversal, final boolean isClearExploratoryExpressions) { for (Quantifier qun : expression.getQuantifiers()) { Reference ref = qun.getRangesOver(); for (RelationalExpression refMember : ref.getAllMemberExpressions()) { - preExploreForRule(refMember, isClearExploratoryExpressions); + preExploreForRule(refMember, traversal, isClearExploratoryExpressions); } ref.setExplored(); PlanContext planContext = new FakePlanContext(); - TestRuleExecution.applyRule(planContext, new FinalizeExpressionsRule(), ref, EvaluationContext.EMPTY); + TestRuleExecution.applyRule(planContext, new FinalizeExpressionsRule(), ref, false, + traversal, EvaluationContext.empty()); pruneInputs(ref.getFinalExpressions(), isClearExploratoryExpressions); } } @@ -230,27 +243,45 @@ private static RelationalExpression costModel(final Reference reference) { @CanIgnoreReturnValue @Nonnull public TestRuleExecution assertYields(RelationalExpression original, RelationalExpression... expected) { + final var root = Reference.initialOf(original); + return assertYields(root, root, original, expected); + } + + @CanIgnoreReturnValue + @Nonnull + public TestRuleExecution assertYields(Reference root, Reference group, RelationalExpression original, + RelationalExpression... expected) { final ImmutableList.Builder expectedListBuilder = ImmutableList.builder(); for (RelationalExpression expression : expected) { // // Copy the expected as the caller can construct the expected from parts of the original. - // we just want to have our own unshared version of it. + // We just want to have our own unshared version of it. // final var copiedExpected = Iterables.getOnlyElement(References.rebaseGraphs(ImmutableList.of(Reference.initialOf(expression)), - Memoizer.noMemoization(PlannerStage.INITIAL), new ToUniqueAliasesTranslationMap(), false)).get(); + Memoizer.noMemoization(PlannerStage.INITIAL), new ToUniqueAliasesTranslationMap(), false)); + + // + // Advance the copied expected expression to the CANONICAL phase + // ensureStage(PlannerStage.CANONICAL, copiedExpected); - expectedListBuilder.add(copiedExpected); + expectedListBuilder.add(copiedExpected.get()); } final var expectedList = expectedListBuilder.build(); + + // + // If the rule that is being tested is an implementation rule, we need to prepare the subgraph underneath + // the group to be tested in a way that the implementation rule can find finalized expressions. + // if (rule instanceof ImplementationCascadesRule) { for (RelationalExpression expression : expectedList) { - preExploreForRule(expression, true); + preExploreForRule(expression, Traversal.withRoot(Reference.initialOf(expression)), + true); pruneInputs(ImmutableList.of(expression), true); } } - TestRuleExecution execution = run(original); + TestRuleExecution execution = run(root, group, original); try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { softly.assertThat(execution.getResult().getAllMemberExpressions()) .hasSize(1 + expectedList.size()) @@ -262,7 +293,15 @@ public TestRuleExecution assertYields(RelationalExpression original, RelationalE @CanIgnoreReturnValue @Nonnull public TestRuleExecution assertYieldsNothing(RelationalExpression original, boolean matched) { - TestRuleExecution execution = run(original); + final var root = Reference.initialOf(original); + return assertYieldsNothing(root, root, original, matched); + } + + @CanIgnoreReturnValue + @Nonnull + public TestRuleExecution assertYieldsNothing(Reference root, Reference group, RelationalExpression original, + boolean matched) { + TestRuleExecution execution = run(root, group, original); try (AutoCloseableSoftAssertions softly = new AutoCloseableSoftAssertions()) { softly.assertThat(execution.hasYielded()) .as("rule should not have yielded new expressions") diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/DecorrelateValuesRuleTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/DecorrelateValuesRuleTest.java index dd46de3cb4..063fd19130 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/DecorrelateValuesRuleTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/DecorrelateValuesRuleTest.java @@ -157,6 +157,10 @@ void multipleValuesDecorrelation() { .addPredicate(fieldPredicate(baseQun, "c", new Comparisons.ValueComparison(Comparisons.Type.NOT_EQUALS, fieldValue(values3, "z")))) .build().buildSelect(); + final Reference group = Reference.initialOf(selectExpression); + final SelectExpression selectExpressionWithExternalCorrelations = join(otherQun, forEach(group)) + .build().buildSelect(); + final SelectExpression expected = GraphExpansion.builder() .addQuantifier(baseQun) .addResultColumn(projectColumn(baseQun, "a")) @@ -168,7 +172,8 @@ void multipleValuesDecorrelation() { .addPredicate(fieldPredicate(baseQun, "c", new Comparisons.ValueComparison(Comparisons.Type.NOT_EQUALS, fieldValue(otherQun, "gamma")))) .build().buildSelect(); - testHelper.assertYields(selectExpression, expected); + testHelper.assertYields(Reference.initialOf(selectExpressionWithExternalCorrelations), group, selectExpression, + expected); } /** diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/SelectMergeRuleTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/SelectMergeRuleTest.java index f2edf858a7..070ba0b668 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/SelectMergeRuleTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/SelectMergeRuleTest.java @@ -557,7 +557,7 @@ void retainNonMergeableChildren() { final Quantifier unionQun = forEach(new LogicalUnionExpression(ImmutableList.of( forEach(selectWithPredicates(t1, ImmutableList.of("a", "b", "c"), fieldPredicate(t1, "b", GREATER_THAN_HELLO))), - forEach(selectWithPredicates(t2, ImmutableList.of("a", "b", "c"), fieldPredicate(t1, "a", EQUALS_42))) + forEach(selectWithPredicates(t2, ImmutableList.of("a", "b", "c"), fieldPredicate(t2, "a", EQUALS_42))) ))); final Quantifier t3 = baseT(); @@ -565,7 +565,7 @@ void retainNonMergeableChildren() { final Quantifier filterGQun = forEach(selectWithPredicates(explodeGQun, ImmutableList.of("two", "three"), fieldPredicate(explodeGQun, "one", new Comparisons.NullComparison(Comparisons.Type.NOT_NULL)))); - final SelectExpression select = join(unionQun, filterGQun) + final SelectExpression select = join(t3, unionQun, filterGQun) .addResultColumn(projectColumn(unionQun, "a")) .addResultColumn(projectColumn(unionQun, "b")) .addResultColumn(projectColumn(unionQun, "c")) @@ -574,7 +574,7 @@ void retainNonMergeableChildren() { .addPredicate(fieldPredicate(unionQun, "b", new Comparisons.ValueComparison(Comparisons.Type.EQUALS, fieldValue(filterGQun, "two")))) .build().buildSelect(); - final SelectExpression merged = join(unionQun, explodeGQun) + final SelectExpression merged = join(t3, unionQun, explodeGQun) .addResultColumn(projectColumn(unionQun, "a")) .addResultColumn(projectColumn(unionQun, "b")) .addResultColumn(projectColumn(unionQun, "c")) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/TestRuleExecution.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/TestRuleExecution.java index a2e9a1cd8b..0ce2bb0ee3 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/TestRuleExecution.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/rules/TestRuleExecution.java @@ -34,7 +34,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.ArrayDeque; -import java.util.Iterator; /** * A helper class for executing a rule during a unit test, without using one of the tasks from the @@ -76,14 +75,16 @@ public T getResultMemberWithClass(@Nonnull Class clazz) { return null; } - public static TestRuleExecution applyRule(@Nonnull PlanContext context, - @Nonnull CascadesRule rule, - @Nonnull Reference group, + public static TestRuleExecution applyRule(@Nonnull final PlanContext context, + @Nonnull final CascadesRule rule, + @Nonnull final Reference group, + final boolean validateNewExpressions, + @Nonnull final Traversal traversal, @Nonnull final EvaluationContext evaluationContext) { boolean ruleMatched = false; boolean hasYielded = false; for (RelationalExpression expression : group.getAllMemberExpressions()) { - final Iterator ruleCalls = + final var ruleCalls = rule.getMatcher() .bindMatches(context.getPlannerConfiguration(), PlannerBindings.newBuilder() @@ -91,7 +92,14 @@ public static TestRuleExecution applyRule(@Nonnull PlanContext context, .build(), expression) .map(bindings -> new CascadesRuleCall(PlannerPhase.REWRITING, context, rule, group, - Traversal.withRoot(group), new ArrayDeque<>(), bindings, evaluationContext)) + traversal, new ArrayDeque<>(), bindings, evaluationContext) { + @Override + protected void validateNewExpression(@Nonnull final RelationalExpression expression) { + if (validateNewExpressions) { + super.validateNewExpression(expression); + } + } + }) .iterator(); while (ruleCalls.hasNext()) { final var ruleCall = ruleCalls.next(); diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index 3c008556e0..f2f40dd55d 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -245,6 +245,7 @@ public void bitmap(YamlTest.Runner runner) throws Exception { } @TestTemplate + @MaintainYamlTestConfig(YamlTestConfigFilters.CORRECT_EXPLAINS) public void recursiveCte(YamlTest.Runner runner) throws Exception { runner.runYamsql("recursive-cte.yamsql"); }