diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java index 217a3bb31cd..8a646219511 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java @@ -34,7 +34,6 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.clientImpl.ClientContext; -import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; import org.apache.accumulo.core.clientImpl.ScannerImpl; import org.apache.accumulo.core.clientImpl.ScannerOptions; import org.apache.accumulo.core.data.ArrayByteSequence; @@ -49,10 +48,10 @@ import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.IteratorBuilder; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.hadoop.io.Text; /** @@ -87,70 +86,6 @@ public class ClientSideIteratorScanner extends ScannerOptions implements Scanner private final Supplier context; private final Supplier tableId; - private class ClientSideIteratorEnvironment implements IteratorEnvironment { - - private final SamplerConfiguration samplerConfig; - private final boolean sampleEnabled; - - ClientSideIteratorEnvironment(boolean sampleEnabled, SamplerConfiguration samplerConfig) { - this.sampleEnabled = sampleEnabled; - this.samplerConfig = samplerConfig; - } - - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.scan; - } - - @Override - public boolean isFullMajorCompaction() { - // The javadocs state this method will throw an ISE when scope is not majc - throw new IllegalStateException( - "Asked about major compaction type when scope is " + getIteratorScope()); - } - - @Override - public boolean isUserCompaction() { - return false; - } - - @Override - public Authorizations getAuthorizations() { - return ClientSideIteratorScanner.this.getAuthorizations(); - } - - @Override - public IteratorEnvironment cloneWithSamplingEnabled() { - return new ClientSideIteratorEnvironment(true, samplerConfig); - } - - @Override - public boolean isSamplingEnabled() { - return sampleEnabled; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return samplerConfig; - } - - @Deprecated(since = "2.1.0") - @Override - public ServiceEnvironment getServiceEnv() { - return new ClientServiceEnvironmentImpl(context.get()); - } - - @Override - public PluginEnvironment getPluginEnv() { - return new ClientServiceEnvironmentImpl(context.get()); - } - - @Override - public TableId getTableId() { - return tableId.get(); - } - } - /** * A class that wraps a Scanner in a SortedKeyValueIterator so that other accumulo iterators can * use it as a source. @@ -295,9 +230,14 @@ public Iterator> iterator() { SortedKeyValueIterator skvi; try { - IteratorEnvironment iterEnv = new ClientSideIteratorEnvironment( - getSamplerConfiguration() != null, getIteratorSamplerConfigurationInternal()); - + ClientIteratorEnvironment.Builder builder = new ClientIteratorEnvironment.Builder() + .withClient(context.get()).withAuthorizations(getAuthorizations()) + .withScope(IteratorScope.scan).withTableId(tableId.get()) + .withSamplerConfiguration(getIteratorSamplerConfigurationInternal()); + if (getSamplerConfiguration() != null) { + builder.withSamplingEnabled(); + } + IteratorEnvironment iterEnv = builder.build(); IteratorBuilder ib = IteratorBuilder.builder(tm.values()).opts(serverSideIteratorOptions).env(iterEnv).build(); diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index be49a9c68ac..38172594afb 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -30,8 +30,9 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.rfile.RFileScannerBuilder.InputArgs; -import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; import org.apache.accumulo.core.clientImpl.ScannerOptions; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; @@ -42,6 +43,7 @@ import org.apache.accumulo.core.data.Column; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration; import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheManagerFactory; @@ -55,6 +57,7 @@ import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.IteratorBuilder; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; @@ -66,6 +69,7 @@ import org.apache.accumulo.core.spi.cache.CacheType; import org.apache.accumulo.core.spi.crypto.CryptoEnvironment; import org.apache.accumulo.core.spi.crypto.CryptoService; +import org.apache.accumulo.core.util.ConfigurationImpl; import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.io.Text; @@ -74,8 +78,46 @@ class RFileScanner extends ScannerOptions implements Scanner { + private static class RFileScannerEnvironmentImpl extends ClientServiceEnvironmentImpl { + + private final Configuration conf; + private final Configuration tableConf; + + public RFileScannerEnvironmentImpl(Opts opts) { + super(null); + conf = new ConfigurationImpl(new ConfigurationCopy(DefaultConfiguration.getInstance())); + ConfigurationCopy tableCC = new ConfigurationCopy(DefaultConfiguration.getInstance()); + if (opts.tableConfig != null) { + opts.tableConfig.forEach(tableCC::set); + } + tableConf = new ConfigurationImpl(tableCC); + } + + @Override + public String getTableName(TableId tableId) throws TableNotFoundException { + Preconditions.checkArgument(tableId == TABLE_ID, "Expected " + TABLE_ID + " obtained" + + " from IteratorEnvironment.getTableId(), but got: " + tableId); + return TABLE_NAME; + } + + @Override + public Configuration getConfiguration() { + return conf; + } + + @Override + public Configuration getConfiguration(TableId tableId) { + Preconditions.checkArgument(tableId == TABLE_ID, "Expected " + TABLE_ID + " obtained" + + " from IteratorEnvironment.getTableId(), but got: " + tableId); + return tableConf; + } + + } + private static final byte[] EMPTY_BYTES = new byte[0]; private static final Range EMPTY_RANGE = new Range(); + private static final String TABLE_NAME = "rfileScanner"; + private static final TableId TABLE_ID = TableId.of(TABLE_NAME); private Range range; private BlockCacheManager blockCacheManager = null; @@ -225,33 +267,6 @@ public void updateScanIteratorOption(String iteratorName, String key, String val super.updateScanIteratorOption(iteratorName, key, value); } - private class IterEnv implements IteratorEnvironment { - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.scan; - } - - @Override - public boolean isFullMajorCompaction() { - return false; - } - - @Override - public Authorizations getAuthorizations() { - return opts.auths; - } - - @Override - public boolean isSamplingEnabled() { - return RFileScanner.this.getSamplerConfiguration() != null; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return RFileScanner.this.getSamplerConfiguration(); - } - } - @Override public Iterator> iterator() { try { @@ -292,15 +307,23 @@ public Iterator> iterator() { EMPTY_BYTES, tableConf); } + ClientIteratorEnvironment.Builder iterEnvBuilder = new ClientIteratorEnvironment.Builder() + .withEnvironment(new RFileScannerEnvironmentImpl(opts)).withAuthorizations(opts.auths) + .withScope(IteratorScope.scan).withTableId(TABLE_ID); + if (getSamplerConfiguration() != null) { + iterEnvBuilder.withSamplerConfiguration(getSamplerConfiguration()); + iterEnvBuilder.withSamplingEnabled(); + } + IteratorEnvironment iterEnv = iterEnvBuilder.build(); try { if (opts.tableConfig != null && !opts.tableConfig.isEmpty()) { var ibEnv = IteratorConfigUtil.loadIterConf(IteratorScope.scan, serverSideIteratorList, serverSideIteratorOptions, tableConf); - var iteratorBuilder = ibEnv.env(new IterEnv()).build(); + var iteratorBuilder = ibEnv.env(iterEnv).build(); iterator = IteratorConfigUtil.loadIterators(iterator, iteratorBuilder); } else { var iteratorBuilder = IteratorBuilder.builder(serverSideIteratorList) - .opts(serverSideIteratorOptions).env(new IterEnv()).build(); + .opts(serverSideIteratorOptions).env(iterEnv).build(); iterator = IteratorConfigUtil.loadIterators(iterator, iteratorBuilder); } } catch (IOException e) { diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java index a03cc811ab8..d64e0181487 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java @@ -34,7 +34,6 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; @@ -54,6 +53,7 @@ import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; import org.apache.accumulo.core.iteratorsImpl.system.SystemIteratorUtil; @@ -66,7 +66,6 @@ import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.accumulo.core.volume.VolumeConfiguration; import org.apache.hadoop.conf.Configuration; @@ -75,105 +74,6 @@ class OfflineIterator implements Iterator> { - static class OfflineIteratorEnvironment implements IteratorEnvironment { - - private final Authorizations authorizations; - private final AccumuloConfiguration conf; - private final boolean useSample; - private final SamplerConfiguration sampleConf; - private final ClientContext context; - private final TableId tableId; - - public OfflineIteratorEnvironment(ClientContext context, TableId tableId, Authorizations auths, - AccumuloConfiguration acuTableConf, boolean useSample, SamplerConfiguration samplerConf) { - this.context = context; - this.tableId = tableId; - this.authorizations = auths; - this.conf = acuTableConf; - this.useSample = useSample; - this.sampleConf = samplerConf; - } - - @Deprecated(since = "2.0.0") - @Override - public AccumuloConfiguration getConfig() { - return conf; - } - - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.scan; - } - - @Override - public boolean isFullMajorCompaction() { - return false; - } - - @Override - public boolean isUserCompaction() { - return false; - } - - private final ArrayList> topLevelIterators = - new ArrayList<>(); - - @Deprecated(since = "2.0.0") - @Override - public void registerSideChannel(SortedKeyValueIterator iter) { - topLevelIterators.add(iter); - } - - @Override - public Authorizations getAuthorizations() { - return authorizations; - } - - SortedKeyValueIterator getTopLevelIterator(SortedKeyValueIterator iter) { - if (topLevelIterators.isEmpty()) { - return iter; - } - ArrayList> allIters = new ArrayList<>(topLevelIterators); - allIters.add(iter); - return new MultiIterator(allIters, false); - } - - @Override - public boolean isSamplingEnabled() { - return useSample; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return sampleConf; - } - - @Override - public IteratorEnvironment cloneWithSamplingEnabled() { - if (sampleConf == null) { - throw new SampleNotPresentException(); - } - return new OfflineIteratorEnvironment(context, tableId, authorizations, conf, true, - sampleConf); - } - - @Deprecated(since = "2.1.0") - @Override - public ServiceEnvironment getServiceEnv() { - return new ClientServiceEnvironmentImpl(context); - } - - @Override - public PluginEnvironment getPluginEnv() { - return new ClientServiceEnvironmentImpl(context); - } - - @Override - public TableId getTableId() { - return tableId; - } - } - private SortedKeyValueIterator iter; private Range range; private KeyExtent currentExtent; @@ -345,9 +245,13 @@ private SortedKeyValueIterator createIterator(KeyExtent extent, MultiIterator multiIter = new MultiIterator(readers, extent); - OfflineIteratorEnvironment iterEnv = - new OfflineIteratorEnvironment(context, tableId, authorizations, tableCC, false, - samplerConfImpl == null ? null : samplerConfImpl.toSamplerConfiguration()); + ClientIteratorEnvironment.Builder iterEnvBuilder = + new ClientIteratorEnvironment.Builder().withAuthorizations(authorizations) + .withScope(IteratorScope.scan).withTableId(tableId).withClient(context); + if (samplerConfImpl != null) { + iterEnvBuilder.withSamplerConfiguration(samplerConfImpl.toSamplerConfiguration()); + } + IteratorEnvironment iterEnv = iterEnvBuilder.build(); byte[] defaultSecurityLabel; ColumnVisibility cv = @@ -360,8 +264,7 @@ private SortedKeyValueIterator createIterator(KeyExtent extent, var iteratorBuilderEnv = IteratorConfigUtil.loadIterConf(IteratorScope.scan, options.serverSideIteratorList, options.serverSideIteratorOptions, tableCC); var iteratorBuilder = iteratorBuilderEnv.env(iterEnv).build(); - return iterEnv - .getTopLevelIterator(IteratorConfigUtil.loadIterators(visFilter, iteratorBuilder)); + return IteratorConfigUtil.loadIterators(visFilter, iteratorBuilder); } @Override diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java index 372a0e49a30..1646a196388 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java @@ -24,6 +24,7 @@ import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; @@ -37,10 +38,7 @@ public interface IteratorEnvironment { * @deprecated since 2.0.0. This is a legacy method used for internal backwards compatibility. */ @Deprecated(since = "2.0.0") - default SortedKeyValueIterator reserveMapFileReader(String mapFileName) - throws IOException { - throw new UnsupportedOperationException(); - } + SortedKeyValueIterator reserveMapFileReader(String mapFileName) throws IOException; /** * @deprecated since 2.0.0. This method was using an unstable non public type. Use @@ -48,40 +46,32 @@ default SortedKeyValueIterator reserveMapFileReader(String mapFileNam */ @Deprecated(since = "2.0.0") default AccumuloConfiguration getConfig() { - throw new UnsupportedOperationException(); + return new ConfigurationCopy(getPluginEnv().getConfiguration()); } /** * Return the executed scope of the Iterator. Value will be one of the following: * {@link IteratorScope#scan}, {@link IteratorScope#minc}, {@link IteratorScope#majc} */ - default IteratorScope getIteratorScope() { - throw new UnsupportedOperationException(); - } + IteratorScope getIteratorScope(); /** * Return true if the compaction is a full major compaction. Will throw IllegalStateException if * {@link #getIteratorScope()} != {@link IteratorScope#majc}. */ - default boolean isFullMajorCompaction() { - throw new UnsupportedOperationException(); - } + boolean isFullMajorCompaction(); /** * @deprecated since 2.0.0. This was an experimental feature and was never tested or documented. */ @Deprecated(since = "2.0.0") - default void registerSideChannel(SortedKeyValueIterator iter) { - throw new UnsupportedOperationException(); - } + void registerSideChannel(SortedKeyValueIterator iter); /** * Return the Scan Authorizations used in this Iterator. Will throw UnsupportedOperationException * if {@link #getIteratorScope()} != {@link IteratorScope#scan}. */ - default Authorizations getAuthorizations() { - throw new UnsupportedOperationException(); - } + Authorizations getAuthorizations(); /** * Returns a new iterator environment object that can be used to create deep copies over sample @@ -113,9 +103,7 @@ default Authorizations getAuthorizations() { * @throws SampleNotPresentException when sampling is not configured for table. * @since 1.8.0 */ - default IteratorEnvironment cloneWithSamplingEnabled() { - throw new UnsupportedOperationException(); - } + IteratorEnvironment cloneWithSamplingEnabled(); /** * There are at least two conditions under which sampling will be enabled for an environment. One @@ -126,27 +114,21 @@ default IteratorEnvironment cloneWithSamplingEnabled() { * @return true if sampling is enabled for this environment. * @since 1.8.0 */ - default boolean isSamplingEnabled() { - throw new UnsupportedOperationException(); - } + boolean isSamplingEnabled(); /** * * @return sampling configuration is sampling is enabled for environment, otherwise returns null. * @since 1.8.0 */ - default SamplerConfiguration getSamplerConfiguration() { - throw new UnsupportedOperationException(); - } + SamplerConfiguration getSamplerConfiguration(); /** * True if compaction was user initiated. * * @since 2.0.0 */ - default boolean isUserCompaction() { - throw new UnsupportedOperationException(); - } + boolean isUserCompaction(); /** * Returns an object containing information about the server where this iterator was run. To @@ -161,9 +143,7 @@ default boolean isUserCompaction() { * {@link #getPluginEnv()} instead because it has better stability guarantees. */ @Deprecated(since = "2.1.0") - default ServiceEnvironment getServiceEnv() { - throw new UnsupportedOperationException(); - } + ServiceEnvironment getServiceEnv(); /** * Returns an object containing information about the server where this iterator was run. To @@ -175,16 +155,12 @@ default ServiceEnvironment getServiceEnv() { * * @since 2.1.0 */ - default PluginEnvironment getPluginEnv() { - return getServiceEnv(); - } + PluginEnvironment getPluginEnv(); /** * Return the table Id associated with this iterator. * * @since 2.0.0 */ - default TableId getTableId() { - throw new UnsupportedOperationException(); - } + TableId getTableId(); } diff --git a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java new file mode 100644 index 00000000000..41a514f19aa --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.iteratorsImpl; + +import static com.google.common.base.Preconditions.checkState; + +import java.io.IOException; +import java.util.Optional; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.PluginEnvironment; +import org.apache.accumulo.core.client.SampleNotPresentException; +import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; + +public class ClientIteratorEnvironment implements IteratorEnvironment { + + public static class Builder { + + private Optional scope = Optional.empty(); + private boolean isFullMajorCompaction = false; + private Optional auths = Optional.empty(); + private boolean isUserCompaction = false; + private Optional tableId = Optional.empty(); + private Optional samplerConfig = Optional.empty(); + private boolean samplingEnabled = false; + protected Optional env = Optional.empty(); + + public Builder withScope(IteratorScope scope) { + checkState(this.scope.isEmpty(), "Scope has already been set"); + this.scope = Optional.of(scope); + return this; + } + + public Builder isFullMajorCompaction() { + this.isFullMajorCompaction = true; + return this; + } + + public Builder withAuthorizations(Authorizations auths) { + checkState(this.auths.isEmpty(), "Authorizations have already been set"); + this.auths = Optional.of(auths); + return this; + } + + public Builder isUserCompaction() { + this.isUserCompaction = true; + return this; + } + + public Builder withTableId(TableId tableId) { + checkState(this.tableId.isEmpty(), "TableId has already been set"); + this.tableId = Optional.of(tableId); + return this; + } + + public Builder withSamplingEnabled() { + this.samplingEnabled = true; + return this; + } + + public Builder withSamplerConfiguration(SamplerConfiguration sc) { + checkState(this.samplerConfig.isEmpty(), "SamplerConfiguration has already been set"); + this.samplerConfig = Optional.ofNullable(sc); + return this; + } + + public ClientIteratorEnvironment.Builder withEnvironment(ClientServiceEnvironmentImpl env) { + checkState(this.env.isEmpty(), "ClientServiceEnvironmentImpl has already been set"); + this.env = Optional.of(env); + return this; + } + + public Builder withClient(AccumuloClient client) { + checkState(this.env.isEmpty(), "ClientServiceEnvironmentImpl has already been set"); + this.env = Optional.of(new ClientServiceEnvironmentImpl((ClientContext) client)); + return this; + } + + public ClientIteratorEnvironment build() { + return new ClientIteratorEnvironment(this); + } + + } + + public static final IteratorEnvironment DEFAULT = new Builder().build(); + + private final Optional scope; + private final boolean isFullMajorCompaction; + private final Optional auths; + private final boolean isUserCompaction; + private final Optional tableId; + private final Optional samplerConfig; + private final boolean samplingEnabled; + private final Optional env; + + private ClientIteratorEnvironment(Builder builder) { + this.scope = builder.scope; + this.isFullMajorCompaction = builder.isFullMajorCompaction; + this.auths = builder.auths; + this.isUserCompaction = builder.isUserCompaction; + this.tableId = builder.tableId; + this.samplerConfig = builder.samplerConfig; + this.env = builder.env; + this.samplingEnabled = builder.samplingEnabled; + } + + /** + * Copy constructor used for enabling sample. Only called from {@link cloneWithSamplingEnabled}. + */ + private ClientIteratorEnvironment(ClientIteratorEnvironment copy) { + this.scope = copy.scope; + this.isFullMajorCompaction = copy.isFullMajorCompaction; + this.auths = copy.auths; + this.isUserCompaction = copy.isUserCompaction; + this.tableId = copy.tableId; + this.samplerConfig = copy.samplerConfig; + this.env = copy.env; + this.samplingEnabled = true; + } + + @Override + @Deprecated(since = "2.0.0") + public SortedKeyValueIterator reserveMapFileReader(String mapFileName) + throws IOException { + throw new UnsupportedOperationException("Feature not supported"); + } + + @Override + public IteratorScope getIteratorScope() { + return scope.orElseThrow(); + } + + @Override + public boolean isFullMajorCompaction() { + if (getIteratorScope() != IteratorScope.majc) { + throw new IllegalStateException("Iterator scope is not majc"); + } + return isFullMajorCompaction; + } + + @Override + @Deprecated(since = "2.0.0") + public void registerSideChannel(SortedKeyValueIterator iter) { + throw new UnsupportedOperationException("Feature not supported"); + } + + @Override + public Authorizations getAuthorizations() { + if (getIteratorScope() != IteratorScope.scan) { + throw new IllegalStateException("Iterator scope is not scan"); + } + return auths.orElseThrow(); + } + + @Override + public IteratorEnvironment cloneWithSamplingEnabled() { + if (samplerConfig.isEmpty()) { + throw new SampleNotPresentException(); + } + return new ClientIteratorEnvironment(this); + } + + @Override + public boolean isSamplingEnabled() { + return this.samplingEnabled; + } + + @Override + public SamplerConfiguration getSamplerConfiguration() { + if (!isSamplingEnabled()) { + return null; + } + return samplerConfig.orElseThrow(); + } + + @Override + public boolean isUserCompaction() { + if (getIteratorScope() == IteratorScope.scan) { + throw new IllegalStateException( + "scan iterator scope is incompatible with a possible user compaction"); + } + return this.isUserCompaction; + } + + @Override + @Deprecated(since = "2.1.0") + public ServiceEnvironment getServiceEnv() { + return env.orElseThrow(); + } + + @Override + public PluginEnvironment getPluginEnv() { + return env.orElseThrow(); + } + + @Override + public TableId getTableId() { + return this.tableId.orElseThrow(); + } + +} diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java index 4b0c533dd1d..0bb58e6bb55 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java @@ -49,7 +49,6 @@ import org.apache.accumulo.core.client.sample.RowSampler; import org.apache.accumulo.core.client.sample.Sampler; -import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.DefaultConfiguration; @@ -74,6 +73,7 @@ import org.apache.accumulo.core.file.rfile.bcfile.BCFile; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; @@ -107,26 +107,6 @@ public class RFileTest { private static final SecureRandom random = new SecureRandom(); - - public static class SampleIE implements IteratorEnvironment { - - private SamplerConfiguration samplerConfig; - - SampleIE(SamplerConfiguration config) { - this.samplerConfig = config; - } - - @Override - public boolean isSamplingEnabled() { - return samplerConfig != null; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return samplerConfig; - } - } - private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); private static final Configuration hadoopConf = new Configuration(); @@ -2068,15 +2048,16 @@ public void testSample() throws IOException { trf.openReader(); - FileSKVIterator sample = - trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf)); + SamplerConfigurationImpl sc = SamplerConfigurationImpl.newSamplerConfig(sampleConf); + + FileSKVIterator sample = trf.reader.getSample(sc); checkSample(sample, sampleData); assertEquals(expectedDataHash, hash(trf.reader)); - SampleIE ie = new SampleIE( - SamplerConfigurationImpl.newSamplerConfig(sampleConf).toSamplerConfiguration()); + IteratorEnvironment ie = new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sc.toSamplerConfiguration()).withSamplingEnabled().build(); for (int i = 0; i < 3; i++) { // test opening and closing deep copies a few times. @@ -2086,8 +2067,10 @@ public void testSample() throws IOException { SortedKeyValueIterator sampleDC1 = sample.deepCopy(ie); SortedKeyValueIterator sampleDC2 = sample.deepCopy(ie); SortedKeyValueIterator sampleDC3 = trf.reader.deepCopy(ie); - SortedKeyValueIterator allDC1 = sampleDC1.deepCopy(new SampleIE(null)); - SortedKeyValueIterator allDC2 = sample.deepCopy(new SampleIE(null)); + SortedKeyValueIterator allDC1 = + sampleDC1.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator allDC2 = + sample.deepCopy(ClientIteratorEnvironment.DEFAULT); assertEquals(expectedDataHash, hash(allDC1)); assertEquals(expectedDataHash, hash(allDC2)); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java b/core/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java deleted file mode 100644 index 802dfe849cc..00000000000 --- a/core/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.core.iterators; - -import java.io.IOException; - -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.DefaultConfiguration; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iteratorsImpl.system.MapFileIterator; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -public class DefaultIteratorEnvironment implements IteratorEnvironment { - - AccumuloConfiguration conf; - Configuration hadoopConf = new Configuration(); - - public DefaultIteratorEnvironment(AccumuloConfiguration conf) { - this.conf = conf; - } - - public DefaultIteratorEnvironment() { - this.conf = DefaultConfiguration.getInstance(); - } - - @Deprecated(since = "2.0.0") - @Override - public SortedKeyValueIterator reserveMapFileReader(String mapFileName) - throws IOException { - FileSystem fs = FileSystem.get(hadoopConf); - return new MapFileIterator(fs, mapFileName, hadoopConf); - } - - @Override - public boolean isSamplingEnabled() { - return false; - } -} diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java index 5a7f6dbcc07..06e8311677a 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.CountingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.junit.jupiter.api.Test; @@ -41,9 +42,8 @@ private static long process(TreeMap sourceMap, TreeMap res SortedMapIterator source = new SortedMapIterator(sourceMap); CountingIterator counter = new CountingIterator(source); FirstEntryInRowIterator feiri = new FirstEntryInRowIterator(); - IteratorEnvironment env = new DefaultIteratorEnvironment(); - feiri.init(counter, iteratorSetting.getOptions(), env); + feiri.init(counter, iteratorSetting.getOptions(), ClientIteratorEnvironment.DEFAULT); feiri.seek(range, Set.of(), false); while (feiri.hasTop()) { diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java index c35cf8f1c6a..3dab1bf667e 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java @@ -25,6 +25,7 @@ import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.RowSampler; import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.junit.jupiter.api.Test; @@ -33,16 +34,9 @@ public class SortedMapIteratorTest { @Test public void testSampleNotPresent() { SortedMapIterator smi = new SortedMapIterator(new TreeMap<>()); - assertThrows(SampleNotPresentException.class, () -> smi.deepCopy(new IteratorEnvironment() { - @Override - public boolean isSamplingEnabled() { - return true; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return new SamplerConfiguration(RowSampler.class.getName()); - } - })); + assertThrows(SampleNotPresentException.class, + () -> smi.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(new SamplerConfiguration(RowSampler.class.getName())) + .withSamplingEnabled().build())); } } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java index 1fe7f3ac524..0a0716006e9 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java @@ -34,8 +34,8 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.BeforeEach; @@ -72,7 +72,7 @@ private static Key newKey(String row, String cf, String cq) { @BeforeEach public void setUp() { columnSliceFilter.describeOptions(); - iteratorEnvironment = new DefaultIteratorEnvironment(); + iteratorEnvironment = ClientIteratorEnvironment.DEFAULT; is = new IteratorSetting(1, ColumnSliceFilter.class); } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java index df757966903..04c64acf6dc 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java @@ -40,7 +40,6 @@ import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.Combiner.ValueIterator; import org.apache.accumulo.core.iterators.CombinerTestUtil; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.LongCombiner; @@ -50,6 +49,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.TypedValueCombiner; import org.apache.accumulo.core.iterators.ValueFormatException; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; @@ -59,29 +59,8 @@ public class CombinerTest { private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); - static class CombinerIteratorEnvironment extends DefaultIteratorEnvironment { - - private IteratorScope scope; - private boolean isFullMajc; - - CombinerIteratorEnvironment(IteratorScope scope, boolean isFullMajc) { - this.scope = scope; - this.isFullMajc = isFullMajc; - } - - @Override - public IteratorScope getIteratorScope() { - return scope; - } - - @Override - public boolean isFullMajorCompaction() { - return isFullMajc; - } - } - static final IteratorEnvironment SCAN_IE = - new CombinerIteratorEnvironment(IteratorScope.scan, false); + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build(); static Key newKey(int row, int colf, int colq, long ts, boolean deleted) { Key k = newKey(row, colf, colq, ts); @@ -884,8 +863,10 @@ public void testDeleteHandling() throws Exception { TreeMap input = new TreeMap<>(); - IteratorEnvironment paritalMajcIe = new CombinerIteratorEnvironment(IteratorScope.majc, false); - IteratorEnvironment fullMajcIe = new CombinerIteratorEnvironment(IteratorScope.majc, true); + IteratorEnvironment paritalMajcIe = + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build(); + IteratorEnvironment fullMajcIe = new ClientIteratorEnvironment.Builder() + .withScope(IteratorScope.majc).isFullMajorCompaction().build(); // keys that aggregate newKeyValue(input, 1, 1, 1, 1, false, 4L, encoder); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java index 996b3df97a5..c1dd3466769 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java @@ -40,9 +40,9 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; import org.apache.accumulo.core.iterators.Filter; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnQualifierFilter; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.accumulo.core.iteratorsImpl.system.VisibilityFilter; @@ -230,19 +230,19 @@ public void test2a() throws IOException { ColumnAgeOffFilter a = new ColumnAgeOffFilter(); assertTrue(a.validateOptions(is.getOptions())); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(902, size(a)); ColumnAgeOffFilter.addTTL(is, new IteratorSetting.Column("a", "b"), 101L); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(102, size(a)); ColumnAgeOffFilter.removeTTL(is, new IteratorSetting.Column("a", "b")); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a = (ColumnAgeOffFilter) a.deepCopy(null); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); @@ -271,19 +271,19 @@ public void test2aNegate() throws IOException { ColumnAgeOffFilter a = new ColumnAgeOffFilter(); assertTrue(a.validateOptions(is.getOptions())); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(98, size(a)); ColumnAgeOffFilter.addTTL(is, new IteratorSetting.Column("a", "b"), 101L); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(898, size(a)); ColumnAgeOffFilter.removeTTL(is, new IteratorSetting.Column("a", "b")); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a = (ColumnAgeOffFilter) a.deepCopy(null); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); @@ -312,19 +312,19 @@ public void test2b() throws IOException { ColumnAgeOffFilter a = new ColumnAgeOffFilter(); assertTrue(a.validateOptions(is.getOptions())); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(902, size(a)); ColumnAgeOffFilter.addTTL(is, new IteratorSetting.Column("negate", "b"), 101L); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(102, size(a)); ColumnAgeOffFilter.removeTTL(is, new IteratorSetting.Column("negate", "b")); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a = (ColumnAgeOffFilter) a.deepCopy(null); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java index fdb34e1fa36..b9ee9c6a3a9 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java @@ -38,9 +38,9 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.file.rfile.RFileTest; import org.apache.accumulo.core.file.rfile.RFileTest.TestRFile; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -51,7 +51,7 @@ public class IndexedDocIteratorTest { private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); private static final byte[] nullByte = {0}; - private static IteratorEnvironment env = new DefaultIteratorEnvironment(); + private static IteratorEnvironment env = ClientIteratorEnvironment.DEFAULT; Text[] columnFamilies; Text[] otherColumnFamilies; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java index 2c3710d9579..7106db27706 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java @@ -33,9 +33,9 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; @@ -45,7 +45,7 @@ public class IntersectingIteratorTest { private static final SecureRandom random = new SecureRandom(); private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); - private static IteratorEnvironment env = new DefaultIteratorEnvironment(); + private static IteratorEnvironment env = ClientIteratorEnvironment.DEFAULT; HashSet docs = new HashSet<>(); Text[] columnFamilies; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java index b0a83759348..fd6c7c6b9b5 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java @@ -34,6 +34,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.junit.jupiter.api.Test; @@ -67,7 +68,7 @@ private LargeRowFilter setupIterator(TreeMap testData, int maxColumns IteratorSetting is = new IteratorSetting(1, LargeRowFilter.class); LargeRowFilter.setMaxColumns(is, maxColumns); lrfi.init(new ColumnFamilySkippingIterator(smi), is.getOptions(), - new RowDeletingIteratorTest.TestIE(scope, false)); + new ClientIteratorEnvironment.Builder().withScope(scope).build()); return lrfi; } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java index 3a558374b4a..8bcf1dac763 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java @@ -33,7 +33,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -67,7 +67,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, ".*2", null, null, null, false); assertTrue(rei.validateOptions(is.getOptions())); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -82,7 +82,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, null, null, null, "amst", false, true); // Should only match hamster rei.validateOptions(is.getOptions()); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -95,7 +95,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, null, "ya.*", null, null, false); assertTrue(rei.validateOptions(is.getOptions())); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -108,7 +108,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, null, null, ".*01", null, false); assertTrue(rei.validateOptions(is.getOptions())); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -121,7 +121,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, null, null, null, ".*at", false); assertTrue(rei.validateOptions(is.getOptions())); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -133,7 +133,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, null, null, ".*ap", false); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertFalse(rei.hasTop()); @@ -142,7 +142,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, "ya.*", null, ".*at", false); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -154,7 +154,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, "ya.*", null, ".*ap", false); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertFalse(rei.hasTop()); @@ -163,7 +163,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, "boo1", null, null, null, false); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -177,7 +177,7 @@ public void test1() throws IOException { // ----------------------------------------------------- is.clearOptions(); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -195,7 +195,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, "hamster", null, "hamster", "hamster", true); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -207,7 +207,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, "ya.*", "hamster", null, true); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -218,9 +218,9 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, "ya.*", "hamster", null, true); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); - rei.deepCopy(new DefaultIteratorEnvironment()); + rei.deepCopy(ClientIteratorEnvironment.DEFAULT); // ----------------------------------------------------- String multiByteText = new String("\u6d67\u6F68\u7067"); @@ -234,7 +234,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, null, null, multiByteRegex, true); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java index 39f0547cff8..0a8ee00998a 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java @@ -32,8 +32,8 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; @@ -41,27 +41,6 @@ public class RowDeletingIteratorTest { - public static class TestIE implements IteratorEnvironment { - - private IteratorScope scope; - private boolean fmc; - - public TestIE(IteratorScope scope, boolean fmc) { - this.scope = scope; - this.fmc = fmc; - } - - @Override - public IteratorScope getIteratorScope() { - return scope; - } - - @Override - public boolean isFullMajorCompaction() { - return fmc; - } - } - Key newKey(String row, String cf, String cq, long time) { return new Key(new Text(row), new Text(cf), new Text(cq), time); } @@ -91,7 +70,8 @@ public void test1() throws Exception { put(tm1, "r2", "cf1", "cq1", 5, "v1"); RowDeletingIterator rdi = new RowDeletingIterator(); - rdi.init(new SortedMapIterator(tm1), null, new TestIE(IteratorScope.scan, false)); + rdi.init(new SortedMapIterator(tm1), null, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); rdi.seek(new Range(), new ArrayList<>(), false); testAssertions(rdi, "r2", "cf1", "cq1", 5, "v1"); @@ -133,7 +113,8 @@ public void test2() throws Exception { put(tm1, "r2", "cf1", "cq1", 5, "v1"); RowDeletingIterator rdi = new RowDeletingIterator(); - rdi.init(new SortedMapIterator(tm1), null, new TestIE(IteratorScope.scan, false)); + rdi.init(new SortedMapIterator(tm1), null, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); rdi.seek(new Range(), new ArrayList<>(), false); testAssertions(rdi, "r1", "cf1", "cq3", 15, "v1"); @@ -175,7 +156,7 @@ public void test3() throws Exception { RowDeletingIterator rdi = new RowDeletingIterator(); rdi.init(new ColumnFamilySkippingIterator(new SortedMapIterator(tm1)), null, - new TestIE(IteratorScope.scan, false)); + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); HashSet cols = new HashSet<>(); cols.add(new ArrayByteSequence("cf1".getBytes(UTF_8))); @@ -206,7 +187,8 @@ public void test4() throws Exception { put(tm1, "r2", "cf1", "cq1", 5, "v1"); RowDeletingIterator rdi = new RowDeletingIterator(); - rdi.init(new SortedMapIterator(tm1), null, new TestIE(IteratorScope.minc, false)); + rdi.init(new SortedMapIterator(tm1), null, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.minc).build()); rdi.seek(new Range(), new ArrayList<>(), false); testAssertions(rdi, "r1", "", "", 10, RowDeletingIterator.DELETE_ROW_VALUE.toString()); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java index 91908a7764f..63da92a45c3 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java @@ -38,26 +38,14 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.IteratorEnvironment; -import org.apache.accumulo.core.iterators.IteratorUtil; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; public class RowEncodingIteratorTest { - private static final class DummyIteratorEnv implements IteratorEnvironment { - @Override - public IteratorUtil.IteratorScope getIteratorScope() { - return IteratorUtil.IteratorScope.scan; - } - - @Override - public boolean isFullMajorCompaction() { - return false; - } - } - private static final class RowEncodingIteratorImpl extends RowEncodingIterator { public static SortedMap decodeRow(Value rowValue) throws IOException { @@ -139,7 +127,8 @@ public void testEncodeAll() throws IOException { RowEncodingIteratorImpl iter = new RowEncodingIteratorImpl(); Map bigBufferOpts = new HashMap<>(); bigBufferOpts.put(RowEncodingIterator.MAX_BUFFER_SIZE_OPT, "3K"); - iter.init(src, bigBufferOpts, new DummyIteratorEnv()); + iter.init(src, bigBufferOpts, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); iter.seek(range, new ArrayList<>(), false); assertTrue(iter.hasTop()); @@ -173,7 +162,8 @@ public void testEncodeSome() throws IOException { RowEncodingIteratorImpl iter = new RowEncodingIteratorImpl(); Map bigBufferOpts = new HashMap<>(); bigBufferOpts.put(RowEncodingIterator.MAX_BUFFER_SIZE_OPT, "1K"); - iter.init(src, bigBufferOpts, new DummyIteratorEnv()); + iter.init(src, bigBufferOpts, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); assertThrows(IllegalArgumentException.class, () -> iter.seek(range, new ArrayList<>(), false)); // IllegalArgumentException should be thrown as we can't fit the whole row into its buffer } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java index a18583efb43..4c106559ecd 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java @@ -39,9 +39,9 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; @@ -205,7 +205,7 @@ public void test1() throws Exception { new ColumnFamilySkippingIterator(new SortedMapIterator(createKeyValues())); RowFilter filter = new SummingRowFilter(); - filter.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter.init(source, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); filter.seek(new Range(), Collections.emptySet(), false); @@ -235,10 +235,10 @@ public void testChainedRowFilters() throws Exception { SortedMapIterator source = new SortedMapIterator(createKeyValues()); RowFilter filter0 = new TrueFilter(); - filter0.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter0.init(source, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); RowFilter filter = new TrueFilter(); - filter.init(filter0, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter.init(filter0, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); filter.seek(new Range(), Collections.emptySet(), false); @@ -251,10 +251,10 @@ public void testFilterConjunction() throws Exception { SortedMapIterator source = new SortedMapIterator(createKeyValues()); RowFilter filter0 = new RowZeroOrOneFilter(); - filter0.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter0.init(source, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); RowFilter filter = new RowOneOrTwoFilter(); - filter.init(filter0, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter.init(filter0, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); filter.seek(new Range(), Collections.emptySet(), false); @@ -266,7 +266,7 @@ public void deepCopyCopiesTheSource() throws Exception { SortedMapIterator source = new SortedMapIterator(createKeyValues()); RowFilter filter = new RowZeroOrOneFilter(); - filter.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter.init(source, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); filter.seek(new Range(), Collections.emptySet(), false); @@ -286,7 +286,7 @@ public void deepCopyCopiesTheSource() throws Exception { } // Make a copy of the original RowFilter - RowFilter copy = (RowFilter) filter.deepCopy(new DefaultIteratorEnvironment()); + RowFilter copy = (RowFilter) filter.deepCopy(ClientIteratorEnvironment.DEFAULT); // Because it's a copy, we should be able to safely seek this one without affecting the original copy.seek(new Range(), Collections.emptySet(), false); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java index 6798b6fecfb..a9c9c92c938 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java @@ -47,6 +47,7 @@ import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.WrappingIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.accumulo.core.iteratorsImpl.system.VisibilityFilter; @@ -619,7 +620,7 @@ public static class ColFamReversingCompactionKeyTransformingIterator @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { - env = new MajCIteratorEnvironmentAdapter(); + env = new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build(); super.init(source, options, env); } } @@ -665,7 +666,7 @@ public static class IllegalVisCompactionKeyTransformingIterator @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { - env = new MajCIteratorEnvironmentAdapter(); + env = new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build(); super.init(source, options, env); } } @@ -695,7 +696,7 @@ public static class BadVisCompactionKeyTransformingIterator @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { - env = new MajCIteratorEnvironmentAdapter(); + env = new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build(); super.init(source, options, env); } } @@ -742,10 +743,4 @@ private void loadTop() { } } - private static class MajCIteratorEnvironmentAdapter implements IteratorEnvironment { - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.majc; - } - } } diff --git a/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java b/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java index de5794607c4..251a3441b36 100644 --- a/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java @@ -39,7 +39,6 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.thrift.IterInfo; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; @@ -139,8 +138,7 @@ public Value getTopValue() { private SortedKeyValueIterator createIter(IteratorScope scope, SortedMapIterator source, AccumuloConfiguration conf) throws IOException { var ibEnv = IteratorConfigUtil.loadIterConf(scope, EMPTY_ITERS, new HashMap<>(), conf); - var iteratorBuilder = - ibEnv.env(new DefaultIteratorEnvironment(conf)).useClassLoader(null).build(); + var iteratorBuilder = ibEnv.env(ClientIteratorEnvironment.DEFAULT).useClassLoader(null).build(); return IteratorConfigUtil.loadIterators(source, iteratorBuilder); } diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java index 408f9242b43..4794c4f713f 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java @@ -33,7 +33,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; -import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; /** * The necessary user-input to invoke a test on a {@link SortedKeyValueIterator}. @@ -49,7 +49,7 @@ public class IteratorTestInput { private final IteratorEnvironment iteratorEnvironment; /** - * Construct an instance of the test input, using {@link SimpleIteratorEnvironment}. + * Construct an instance of the test input, using {@link ClientIteratorEnvironment}. * * @param iteratorClass The class for the iterator to test. * @param iteratorOptions Options, if any, to provide to the iterator ({@link IteratorSetting}'s @@ -61,7 +61,7 @@ public class IteratorTestInput { public IteratorTestInput(Class> iteratorClass, Map iteratorOptions, Range range, SortedMap input) { this(iteratorClass, iteratorOptions, range, input, Collections.emptySet(), false, - new SimpleIteratorEnvironment()); + ClientIteratorEnvironment.DEFAULT); } /** @@ -93,7 +93,7 @@ public IteratorTestInput(Class> iter * @param families Column families passed to {@link SortedKeyValueIterator#seek}. * @param inclusive Whether the families are inclusive or exclusive. * @param iterEnv An optional provided {@link IteratorEnvironment}. - * {@link SimpleIteratorEnvironment} will be used if null. + * {@link ClientIteratorEnvironment} will be used if null. */ public IteratorTestInput(Class> iteratorClass, Map iteratorOptions, Range range, SortedMap input, @@ -108,7 +108,7 @@ public IteratorTestInput(Class> iter this.input = Collections.unmodifiableSortedMap(requireNonNull(input)); this.families = Collections.unmodifiableCollection(requireNonNull(families)); this.inclusive = inclusive; - this.iteratorEnvironment = iterEnv == null ? new SimpleIteratorEnvironment() : iterEnv; + this.iteratorEnvironment = iterEnv == null ? ClientIteratorEnvironment.DEFAULT : iterEnv; } public Class> getIteratorClass() { diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java deleted file mode 100644 index 0c89197aed8..00000000000 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.iteratortest.environments; - -import org.apache.accumulo.core.iterators.IteratorEnvironment; - -/** - * A simple implementation of {@link IteratorEnvironment} which is unimplemented. - */ -public class SimpleIteratorEnvironment implements IteratorEnvironment { - - @Override - public boolean isSamplingEnabled() { - return false; - } - -} diff --git a/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java b/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java index 63970b2943a..a1b657b5628 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java +++ b/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.Map; +import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; @@ -79,6 +80,30 @@ public TabletIteratorEnvironment(ServerContext context, IteratorScope scope, this.topLevelIterators = new ArrayList<>(); } + public TabletIteratorEnvironment(ServerContext context, IteratorScope scope, + AccumuloConfiguration tableConfig, TableId tableId, SamplerConfigurationImpl samplerConfig) { + if (scope == IteratorScope.majc) { + throw new IllegalArgumentException("must set if compaction is full"); + } + + this.context = context; + this.serviceEnvironment = new ServiceEnvironmentImpl(context); + this.scope = scope; + this.trm = null; + this.tableConfig = tableConfig; + this.tableId = tableId; + this.fullMajorCompaction = false; + this.userCompaction = false; + this.authorizations = Authorizations.EMPTY; + if (samplerConfig != null) { + enableSampleForDeepCopy = true; + this.samplerConfig = samplerConfig.toSamplerConfiguration(); + } else { + enableSampleForDeepCopy = false; + } + this.topLevelIterators = new ArrayList<>(); + } + public TabletIteratorEnvironment(ServerContext context, IteratorScope scope, AccumuloConfiguration tableConfig, TableId tableId, ScanFileManager trm, Map files, Authorizations authorizations, @@ -236,4 +261,9 @@ public ServiceEnvironment getServiceEnv() { public TableId getTableId() { return tableId; } + + @Override + public PluginEnvironment getPluginEnv() { + return serviceEnvironment; + } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java b/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java index 974617d5e84..faa24bdf736 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java @@ -34,8 +34,8 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.DevNull; -import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection; import org.apache.accumulo.server.replication.proto.Replication.Status; import org.junit.jupiter.api.BeforeEach; @@ -48,13 +48,6 @@ public class StatusCombinerTest { private Key key; private Status.Builder builder; - private static class TestIE implements IteratorEnvironment { - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.scan; - } - } - @BeforeEach public void initCombiner() throws IOException { key = new Key(); @@ -62,7 +55,8 @@ public void initCombiner() throws IOException { builder = Status.newBuilder(); IteratorSetting cfg = new IteratorSetting(50, StatusCombiner.class); Combiner.setColumns(cfg, Collections.singletonList(new Column(StatusSection.NAME))); - combiner.init(new DevNull(), cfg.getOptions(), new TestIE()); + combiner.init(new DevNull(), cfg.getOptions(), + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); } @Test diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java index eefa5c1252b..5bd26d5b74f 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java @@ -38,7 +38,6 @@ import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.RowSampler; import org.apache.accumulo.core.client.sample.Sampler; -import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; @@ -49,7 +48,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException; @@ -59,6 +58,7 @@ import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.TableConfiguration; +import org.apache.accumulo.server.iterators.TabletIteratorEnvironment; import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -71,29 +71,6 @@ @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input") public class InMemoryMapTest extends WithTestNames { - private static class SampleIE implements IteratorEnvironment { - - private final SamplerConfiguration sampleConfig; - - public SampleIE() { - this.sampleConfig = null; - } - - public SampleIE(SamplerConfigurationImpl sampleConfig) { - this.sampleConfig = sampleConfig.toSamplerConfiguration(); - } - - @Override - public boolean isSamplingEnabled() { - return sampleConfig != null; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return sampleConfig; - } - } - public static ServerContext getServerContext() { Configuration hadoopConf = new Configuration(); ServerContext context = EasyMock.createMock(ServerContext.class); @@ -319,7 +296,9 @@ public void test6() throws Exception { mutate(imm, "r1", "foo:cq5", 3, "bar5"); - SortedKeyValueIterator dc = ski1.deepCopy(new SampleIE()); + SortedKeyValueIterator dc = + ski1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); ski1.seek(new Range(newKey("r1", "foo:cq1", 3), null), Set.of(), false); testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1"); @@ -371,8 +350,9 @@ private void deepCopyAndDelete(int interleaving, boolean interrupt) throws Excep } } - SortedKeyValueIterator dc = ski1.deepCopy(new SampleIE()); - + SortedKeyValueIterator dc = + ski1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); if (interleaving == 2) { imm.delete(0); if (interrupt) { @@ -523,7 +503,9 @@ public void testLocalityGroups() throws Exception { MemoryIterator iter1 = imm.skvIterator(null); seekLocalityGroups(iter1); - SortedKeyValueIterator dc1 = iter1.deepCopy(new SampleIE()); + SortedKeyValueIterator dc1 = + iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); seekLocalityGroups(dc1); assertEquals(10, imm.getNumEntries()); @@ -576,12 +558,27 @@ public void testSample() throws Exception { MemoryIterator iter1 = imm.skvIterator(sampleConfig); MemoryIterator iter2 = imm.skvIterator(null); - SortedKeyValueIterator iter0dc1 = iter0.deepCopy(new SampleIE()); - SortedKeyValueIterator iter0dc2 = iter0.deepCopy(new SampleIE(sampleConfig)); - SortedKeyValueIterator iter1dc1 = iter1.deepCopy(new SampleIE()); - SortedKeyValueIterator iter1dc2 = iter1.deepCopy(new SampleIE(sampleConfig)); - SortedKeyValueIterator iter2dc1 = iter2.deepCopy(new SampleIE()); - SortedKeyValueIterator iter2dc2 = iter2.deepCopy(new SampleIE(sampleConfig)); + SortedKeyValueIterator iter0dc1 = + iter0.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter0dc2 = iter0.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); + SortedKeyValueIterator iter1dc1 = + iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter1dc2 = iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); + SortedKeyValueIterator iter2dc1 = + iter2.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter2dc2 = iter2.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); assertEquals(expectedNone, readAll(iter0)); assertEquals(expectedNone, readAll(iter0dc1)); @@ -605,12 +602,27 @@ public void testSample() throws Exception { assertEquals(expectedSample, readAll(iter1dc2)); assertEquals(expectedSample, readAll(iter2dc2)); - SortedKeyValueIterator iter0dc3 = iter0.deepCopy(new SampleIE()); - SortedKeyValueIterator iter0dc4 = iter0.deepCopy(new SampleIE(sampleConfig)); - SortedKeyValueIterator iter1dc3 = iter1.deepCopy(new SampleIE()); - SortedKeyValueIterator iter1dc4 = iter1.deepCopy(new SampleIE(sampleConfig)); - SortedKeyValueIterator iter2dc3 = iter2.deepCopy(new SampleIE()); - SortedKeyValueIterator iter2dc4 = iter2.deepCopy(new SampleIE(sampleConfig)); + SortedKeyValueIterator iter0dc3 = + iter0.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter0dc4 = iter0.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); + SortedKeyValueIterator iter1dc3 = + iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter1dc4 = iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); + SortedKeyValueIterator iter2dc3 = + iter2.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter2dc4 = iter2.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); assertEquals(expectedNone, readAll(iter0dc3)); assertEquals(expectedNone, readAll(iter0dc4)); @@ -667,7 +679,9 @@ private void runInterruptSampleTest(boolean deepCopy, boolean delete, boolean dc } if (deepCopy) { - iter = iter.deepCopy(new SampleIE(sampleConfig1)); + iter = iter.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"), + sampleConfig1)); } if (delete && dcAfterDelete) { @@ -768,7 +782,10 @@ public void testDeferredSamplerCreation() throws Exception { iter.seek(new Range(), Set.of(), false); assertEquals(expectedSample, readAll(iter)); - SortedKeyValueIterator dc = iter.deepCopy(new SampleIE(sampleConfig2)); + SortedKeyValueIterator dc = iter.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig2)); dc.seek(new Range(), Set.of(), false); assertEquals(expectedSample, readAll(dc)); diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java index 63ec7fcf113..e34747e3aea 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java @@ -52,7 +52,6 @@ import org.apache.accumulo.core.dataImpl.thrift.IterInfo; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVIterator; -import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; @@ -73,6 +72,7 @@ import org.apache.accumulo.server.cli.ServerUtilOpts; import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.iterators.TabletIteratorEnvironment; import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; @@ -103,8 +103,6 @@ static class CollectOptions extends ServerUtilOpts { String columns; } - static class TestEnvironment implements IteratorEnvironment {} - public static void main(String[] args) throws Exception { final CollectOptions opts = new CollectOptions(); @@ -429,8 +427,8 @@ private static void reportHdfsBlockLocations(ServerContext context, List createScanIterator(KeyExtent ke, Collection> mapfiles, Authorizations authorizations, byte[] defaultLabels, HashSet columnSet, List ssiList, - Map> ssio, boolean useTableIterators, TableConfiguration conf) - throws IOException { + Map> ssio, boolean useTableIterators, TableConfiguration conf, + ServerContext context) throws IOException { SortedMapIterator smi = new SortedMapIterator(new TreeMap<>()); @@ -449,7 +447,9 @@ private static SortedKeyValueIterator createScanIterator(KeyExtent ke if (useTableIterators) { var ibEnv = IteratorConfigUtil.loadIterConf(IteratorScope.scan, ssiList, ssio, conf); - var iteratorBuilder = ibEnv.env(new TestEnvironment()).useClassLoader("test").build(); + TabletIteratorEnvironment iterEnv = + new TabletIteratorEnvironment(context, IteratorScope.scan, conf, ke.tableId()); + var iteratorBuilder = ibEnv.env(iterEnv).useClassLoader("test").build(); return IteratorConfigUtil.loadIterators(visFilter, iteratorBuilder); } return visFilter; @@ -506,7 +506,7 @@ private static int readFilesUsingIterStack(VolumeManager fs, ServerContext conte Map> emptySsio = Collections.emptyMap(); TableConfiguration tconf = context.getTableConfiguration(ke.tableId()); reader = createScanIterator(ke, readers, auths, new byte[] {}, new HashSet<>(), emptyIterinfo, - emptySsio, useTableIterators, tconf); + emptySsio, useTableIterators, tconf, context); HashSet columnSet = createColumnBSS(columns); diff --git a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java index 7d8b21519f0..0bbb6a85139 100644 --- a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java +++ b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java @@ -32,11 +32,11 @@ import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.LongCombiner; import org.apache.accumulo.core.iterators.user.SummingCombiner; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.iteratortest.IteratorTestBase; import org.apache.accumulo.iteratortest.IteratorTestInput; import org.apache.accumulo.iteratortest.IteratorTestOutput; import org.apache.accumulo.iteratortest.IteratorTestParameters; -import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment; /** * Iterator test harness tests for SummingCombiner @@ -48,14 +48,8 @@ public class SummingCombinerTest extends IteratorTestBase { @Override protected Stream parameters() { - var env = new SimpleIteratorEnvironment() { - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.majc; - } - }; - var input = - new IteratorTestInput(SummingCombiner.class, createOpts(), new Range(), INPUT_DATA, env); + var input = new IteratorTestInput(SummingCombiner.class, createOpts(), new Range(), INPUT_DATA, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build()); var expectedOutput = new IteratorTestOutput(OUTPUT_DATA); return builtinTestCases().map(test -> test.toParameters(input, expectedOutput)); }